{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "hw12_reinforcement_learning_english_version.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "de3a153737af485ea436d7e8393d8248": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6345f8926212465291c04587353161f1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3aa84c8c097d4858a0c38f18dec8b060",
              "IPY_MODEL_6647e68cf064416ca593b990bed81edf"
            ]
          }
        },
        "6345f8926212465291c04587353161f1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3aa84c8c097d4858a0c38f18dec8b060": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a25de7edbbee47cc8094f43125efa39b",
            "_dom_classes": [],
            "description": "Total:  86.3, Final:  0.0: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 400,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 400,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a0bb5061ca0946ab89a89f9abadcebc8"
          }
        },
        "6647e68cf064416ca593b990bed81edf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d99d8e6bcbe0445eb7eddbfe31277635",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 400/400 [11:36&lt;00:00,  1.74s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5af1c6579f0f4c2eb03df4063749569e"
          }
        },
        "a25de7edbbee47cc8094f43125efa39b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a0bb5061ca0946ab89a89f9abadcebc8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d99d8e6bcbe0445eb7eddbfe31277635": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5af1c6579f0f4c2eb03df4063749569e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fp30SB4bxeQb"
      },
      "source": [
        "# **Homework 12 - Reinforcement Learning**\n",
        "\n",
        "If you have any problem, e-mail us at ntu-ml-2021spring-ta@googlegroups.com\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yXsnCWPtWSNk"
      },
      "source": [
        "## Preliminary work\n",
        "\n",
        "First, we need to install all necessary packages.\n",
        "One of them, gym, builded by OpenAI, is a toolkit for developing Reinforcement Learning algorithm. Other packages are for visualization in colab."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5e2bScpnkVbv",
        "outputId": "52198e39-e2a2-4ea2-a1f3-4ba9545476d7"
      },
      "source": [
        "!apt update\n",
        "!apt install python-opengl xvfb -y\n",
        "!pip install gym[box2d]==0.18.3 pyvirtualdisplay tqdm numpy==1.19.5 torch==1.8.1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Hit:1 http://security.ubuntu.com/ubuntu bionic-security InRelease\n",
            "Ign:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "Hit:3 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease\n",
            "Ign:4 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "Hit:5 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "Hit:6 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
            "Hit:7 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease\n",
            "Hit:8 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "Hit:9 http://archive.ubuntu.com/ubuntu bionic-updates InRelease\n",
            "Hit:10 http://archive.ubuntu.com/ubuntu bionic-backports InRelease\n",
            "Hit:11 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n",
            "Hit:12 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease\n",
            "Hit:13 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "86 packages can be upgraded. Run 'apt list --upgradable' to see them.\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "python-opengl is already the newest version (3.1.0+dfsg-1).\n",
            "xvfb is already the newest version (2:1.19.6-1ubuntu4.9).\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-460\n",
            "Use 'apt autoremove' to remove it.\n",
            "0 upgraded, 0 newly installed, 0 to remove and 86 not upgraded.\n",
            "Requirement already satisfied: gym[box2d] in /usr/local/lib/python3.7/dist-packages (0.17.3)\n",
            "Requirement already satisfied: pyvirtualdisplay in /usr/local/lib/python3.7/dist-packages (2.1)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (4.41.1)\n",
            "Requirement already satisfied: numpy>=1.10.4 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.19.5)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.4.1)\n",
            "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.5.0)\n",
            "Requirement already satisfied: cloudpickle<1.7.0,>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.3.0)\n",
            "Requirement already satisfied: box2d-py~=2.3.5; extra == \"box2d\" in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (2.3.8)\n",
            "Requirement already satisfied: EasyProcess in /usr/local/lib/python3.7/dist-packages (from pyvirtualdisplay) (0.3)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from pyglet<=1.5.0,>=1.4.0->gym[box2d]) (0.16.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M_-i3cdoYsks"
      },
      "source": [
        "\n",
        "Next, set up virtual display，and import all necessaary packages."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nl2nREINDLiw"
      },
      "source": [
        "%%capture\n",
        "from pyvirtualdisplay import Display\n",
        "virtual_display = Display(visible=0, size=(1400, 900))\n",
        "virtual_display.start()\n",
        "\n",
        "%matplotlib inline\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from IPython import display\n",
        "\n",
        "import numpy as np\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torch.nn.functional as F\n",
        "from torch.distributions import Categorical\n",
        "from tqdm.notebook import tqdm"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CaEJ8BUCpN9P"
      },
      "source": [
        "# Warning ! Do not revise random seed !!!\n",
        "# Your submission on JudgeBoi will not reproduce your result !!!\n",
        "Make your HW result to be reproducible.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fV9i8i2YkRbO"
      },
      "source": [
        "seed = 543 # Do not change this\n",
        "def fix(env, seed):\n",
        "  env.seed(seed)\n",
        "  env.action_space.seed(seed)\n",
        "  torch.manual_seed(seed)\n",
        "  torch.cuda.manual_seed(seed)\n",
        "  torch.cuda.manual_seed_all(seed)\n",
        "  np.random.seed(seed)\n",
        "  random.seed(seed)\n",
        "  torch.set_deterministic(True)\n",
        "  torch.backends.cudnn.benchmark = False\n",
        "  torch.backends.cudnn.deterministic = True"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "He0XDx6bzjgC"
      },
      "source": [
        "Last, call gym and build an [Lunar Lander](https://gym.openai.com/envs/LunarLander-v2/) environment."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N_4-xJcbBt09"
      },
      "source": [
        "%%capture\n",
        "import gym\n",
        "import random\n",
        "env = gym.make('LunarLander-v2')\n",
        "fix(env, seed) # fix the environment Do not revise this !!!"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NrkVvTrvWZ5H"
      },
      "source": [
        "## What Lunar Lander？\n",
        "\n",
        "“LunarLander-v2”is to simulate the situation when the craft lands on the surface of the moon.\n",
        "\n",
        "This task is to enable the craft to land \"safely\" at the pad between the two yellow flags.\n",
        "> Landing pad is always at coordinates (0,0).\n",
        "> Coordinates are the first two numbers in state vector.\n",
        "\n",
        "![](https://gym.openai.com/assets/docs/aeloop-138c89d44114492fd02822303e6b4b07213010bb14ca5856d2d49d6b62d88e53.svg)\n",
        "\n",
        "\"LunarLander-v2\" actually includes \"Agent\" and \"Environment\". \n",
        "\n",
        "In this homework, we will utilize the function `step()` to control the action of \"Agent\". \n",
        "\n",
        "Then `step()` will return the observation/state and reward given by the \"Environment\"."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bIbp82sljvAt"
      },
      "source": [
        "### Observation / State\n",
        "\n",
        "First, we can take a look at what an Observation / State looks like."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rsXZra3N9R5T",
        "outputId": "a36868de-bbbc-4de9-815b-0b43fc012c96"
      },
      "source": [
        "print(env.observation_space)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Box(-inf, inf, (8,), float32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ezdfoThbAQ49"
      },
      "source": [
        "\n",
        "`Box(8,)`means that observation is an 8-dim vector\n",
        "### Action\n",
        "\n",
        "Actions can be taken by looks like"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p1k4dIrBAaKi",
        "outputId": "80c453ee-539f-4e40-c5d8-8e9dc8fffaef"
      },
      "source": [
        "print(env.action_space)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Discrete(4)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dejXT6PHBrPn"
      },
      "source": [
        "`Discrete(4)` implies that there are four kinds of actions can be taken by agent.\n",
        "- 0 implies the agent will not take any actions\n",
        "- 2 implies the agent will accelerate downward\n",
        "- 1, 3 implies the agent will accelerate left and right\n",
        "\n",
        "Next, we will try to make the agent interact with the environment. \n",
        "Before taking any actions, we recommend to call `reset()` function to reset the environment. Also, this function will return the initial state of the environment."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pi4OmrmZgnWA",
        "outputId": "2635bdfb-a4dc-442b-a21a-f57af67edc4b"
      },
      "source": [
        "initial_state = env.reset()\n",
        "print(initial_state)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[ 0.00396109  1.4083536   0.40119505 -0.11407257 -0.00458307 -0.09087662\n",
            "  0.          0.        ]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uBx0mEqqgxJ9"
      },
      "source": [
        "Then, we try to get a random action from the agent's action space."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vxkOEXRKgizt",
        "outputId": "de93c740-f01c-464e-f436-a2b59e7dc7e5"
      },
      "source": [
        "random_action = env.action_space.sample()\n",
        "print(random_action)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mns-bO01g0-J"
      },
      "source": [
        "More, we can utilize `step()` to make agent act according to the randomly-selected `random_action`.\n",
        "The `step()` function will return four values:\n",
        "- observation / state\n",
        "- reward\n",
        "- done (True/ False)\n",
        "- Other information"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E_WViSxGgIk9"
      },
      "source": [
        "observation, reward, done, info = env.step(random_action)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yK7r126kuCNp",
        "outputId": "2f3363d9-5bc3-4ba5-86f2-1c4abc89f179"
      },
      "source": [
        "print(done)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "False\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GKdS8vOihxhc"
      },
      "source": [
        "### Reward\n",
        "\n",
        "\n",
        "> Landing pad is always at coordinates (0,0). Coordinates are the first two numbers in state vector. Reward for moving from the top of the screen to landing pad and zero speed is about 100..140 points. If lander moves away from landing pad it loses reward back. Episode finishes if the lander crashes or comes to rest, receiving additional -100 or +100 points. Each leg ground contact is +10. Firing main engine is -0.3 points each frame. Solved is 200 points. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vxQNs77hi0_7",
        "outputId": "4633d678-be4f-4f52-8f91-1b6681642580"
      },
      "source": [
        "print(reward)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-0.8588900517154912\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Mhqp6D-XgHpe"
      },
      "source": [
        "### Random Agent\n",
        "In the end, before we start training, we can see whether a random agent can successfully land the moon or not."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        },
        "id": "Y3G0bxoccelv",
        "outputId": "11ad28c1-058b-4243-bf35-1fdfbb60be9e"
      },
      "source": [
        "env.reset()\n",
        "\n",
        "img = plt.imshow(env.render(mode='rgb_array'))\n",
        "\n",
        "done = False\n",
        "while not done:\n",
        "    action = env.action_space.sample()\n",
        "    observation, reward, done, _ = env.step(action)\n",
        "\n",
        "    img.set_data(env.render(mode='rgb_array'))\n",
        "    display.display(plt.gcf())\n",
        "    display.clear_output(wait=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAd2ElEQVR4nO3de3RW9b3n8fc39xCu4RLC3SCtRYvcL8taKD0cKWtmsC0qTr3AaCnWarvmTOfomTVHzzmrx1W0doZlh5baKrQVag9eWBZQBI8VKSAochWIIVxyAgECgYDkxnf+eHbCIwnk9iRPdvJ5rfWs7P3bez/7+wvP82Hn9+z9bHN3REQkPBLiXYCIiDSOgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREKmxYLbzKab2T4zyzWzx1pqPyIiHY21xHncZpYI7AemAUeBD4C73X1PzHcmItLBtNQR93gg193z3L0cWA7MbKF9iYh0KEkt9Lz9gSNR80eBCVdb2cx0+aaIyBXc3epqb6ngrpeZzQPmxWv/IiJh1VLBXQAMjJofELTVcPfFwGLQEbeISGO01Bj3B8AwM7vOzFKA2cDKFtqXiEiH0iJH3O5eaWY/AN4EEoHfuvvultiXiEhH0yKnAza6CA2ViIjUcrUPJ3XlpIhIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREKmWfecNLN84BxQBVS6+1gzywT+CAwB8oE73f1088oUEZFqsTji/pq7j3T3scH8Y8A6dx8GrAvmRUQkRlpiqGQmsCSYXgLc3gL7EBHpsJob3A68ZWbbzGxe0Jbl7oXB9DEgq5n7EBGRKM0a4wa+4u4FZtYHWGtmn0QvdHc3M69rwyDo59W1TERErs7c68zVxj+R2ZNAKfBdYIq7F5pZNvDv7v7FeraNTREiIu2Iu1td7U0eKjGzDDPrUj0N/C2wC1gJ3B+sdj/welP3ISIitTX5iNvMcoBXg9kk4CV3/4mZ9QReBgYBh4icDlhcz3PpiFtE5ApXO+KO2VBJcyi4RURqi/lQiYiIxIeCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiFTb3Cb2W/NrMjMdkW1ZZrZWjM7EPzsEbSbmS00s1wz22Fmo1uyeBGRjqghR9wvAtOvaHsMWOfuw4B1wTzAN4BhwWMesCg2ZYqISLV6g9vd/wIUX9E8E1gSTC8Bbo9qX+oRm4DuZpYdq2JFRKTpY9xZ7l4YTB8DsoLp/sCRqPWOBm21mNk8M9tqZlubWIOISIeU1NwncHc3M2/CdouBxQBN2V5EpKNq6hH38eohkOBnUdBeAAyMWm9A0CYiIjHS1OBeCdwfTN8PvB7Vfl9wdslEoCRqSEVERGLA3K89SmFmy4ApQC/gOPAE8BrwMjAIOATc6e7FZmbAc0TOQrkAzHX3esewNVQiIlKbu1td7fUGd2tQcIuI1Ha14NaVkyIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQmZeoPbzH5rZkVmtiuq7UkzKzCz7cFjRtSyx80s18z2mdltLVW4iEhH1ZCbBX8VKAWWuvtNQduTQKm7P3PFusOBZcB4oB/wNvAFd6+qZx+656SIyBWafM9Jd/8LUNzA/cwElrt7mbsfBHKJhLiIiMRIc8a4f2BmO4KhlB5BW3/gSNQ6R4O2WsxsnpltNbOtzahBRKTDaWpwLwKGAiOBQuBnjX0Cd1/s7mPdfWwTaxAR6ZCaFNzuftzdq9z9EvBrLg+HFAADo1YdELSJiEiMNCm4zSw7avabQPUZJyuB2WaWambXAcOALc0rUUREoiXVt4KZLQOmAL3M7CjwBDDFzEYCDuQD3wNw991m9jKwB6gEHq7vjBIREWmcek8HbJUidDqgiEgtTT4dUERE2hYFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjI1BvcZjbQzN4xsz1mttvMfhi0Z5rZWjM7EPzsEbSbmS00s1wz22Fmo1u6EyIiHUlDjrgrgb9z9+HAROBhMxsOPAasc/dhwLpgHuAbRO7uPgyYByyKedUiIh1YvcHt7oXu/mEwfQ7YC/QHZgJLgtWWALcH0zOBpR6xCehuZtkxr1xEpINq1Bi3mQ0BRgGbgSx3LwwWHQOygun+wJGozY4GbVc+1zwz22pmWxtZs4hIh9bg4DazzsAK4EfufjZ6mbs74I3Zsbsvdvex7j62MduJiHR0DQpuM0smEtp/cPdXgubj1UMgwc+ioL0AGBi1+YCgTUREYqAhZ5UY8Btgr7s/G7VoJXB/MH0/8HpU+33B2SUTgZKoIRUREWkmi4xyXGMFs68A7wE7gUtB8z8QGed+GRgEHALudPfiIOifA6YDF4C57n7NcWwza9Qwi4hIR+DuVld7vcHdGhTcIiK1XS24deWkiEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQqYhNwseaGbvmNkeM9ttZj8M2p80swIz2x48ZkRt87iZ5ZrZPjO7rSU7ICLS0TTkZsHZQLa7f2hmXYBtwO3AnUCpuz9zxfrDgWXAeKAf8DbwBXevusY+dM9JEZErNPmek+5e6O4fBtPngL1A/2tsMhNY7u5l7n4QyCUS4iIiEgONGuM2syHAKGBz0PQDM9thZr81sx5BW3/gSNRmR7l20IsA8K//+j1++lO46SYYPhz69Yt3Ra1vypQpvPjiF5kxA268EW64ARIT412VtDVJDV3RzDoDK4AfuftZM1sE/Avgwc+fAf+tEc83D5jXuHKlPfvyl3PIzoapUyPzhYWwZ09kes0ayM0Fdzh2DKquOvAWbr1792b8+FJuvDEyX1kJGzdCRQUcPQqvvRZpLymBc+fiV6fEV4OC28ySiYT2H9z9FQB3Px61/NfAG8FsATAwavMBQdvnuPtiYHGwvca4pYYFo3r9+l0+6v7a1yKhXVUFb74Jn30WCfbf/z5+dbak6t9BcjJMnhyZdod77olM79oF+/ZFppcuhePHaz+HtF8NOavEgN8Ae9392aj27KjVvgnsCqZXArPNLNXMrgOGAVtiV7J0RJcuRUK7shIuXIDz5yPh3ZFU/8dVVQUXL0Z+B+fPR3430rE05Ij7FuBeYKeZbQ/a/gG428xGEhkqyQe+B+Duu83sZWAPUAk8fK0zSkSiuUceEBka2B684t58E/LyIsuKi9t/WFX/HiorYf16KC+HggJYuTKyvLS04/3HJZfVG9zuvgGo65SUVdfY5ifAT5pRl3RApaXw5z9Hhj8uXYqM4Z44Ee+qWt/27fDrX8OhQ5Hfw+HD7f8/KmmcBn84KdLSDh+GJ5+MdxXx9+yzsHVrvKuQtkyXvIuIhIyCW0QkZDRUIhITRkJC5EoZ9yrq+yoJkeZQcIvEwLBhX2Xw4LEAHDu2l5KSQgAuXDjDqVMH41matEMKbpFmMjP69x/Bl/rOpFNyJqd7H+R8eeR0mM/KT3Pq7KcAVFSUkZf3PpeCU0SKiw9TWXkxbnVLeCm4RZopK+uL9On2RbqnDSIxIYX05MyaZRVVF7mYeRqAKi9nUN9xuEeC+/ipPZRVnAfg8OGtFBTsbP3iJZQU3CLN1KVLHzql9iYxIaXWsuTENJITL19k3C11UM10/67jqbpURtH5XRQVHWiVWqV9UHCLNENiYgpDhkwgK+PLDVrf7PK1bF1TI1/EcuZifkuUJu2YTgcUaYYePQbSNb0fyYlp8S5FOhAFt8hVdE1KYnT37jXf99A5LY2BPXuSknT5D9WePQfTNa0/SQkKbmk9GioRqUPXpCS+m5PDdRkZ/LmwkLVFRYzOyaFrejonz53jw7w8SEhh0MAx9MkYHu9ypYPREbdIHbLS0hjUqRPJCQnc3L07iWY1p/FVVUUusBk8eCz9e4wmNalbTPfdtWsWSUk6gperU3CL1OFAaSk7S0o4cO4cv87Lo8KNqi4j8B5jOXz2EhVVVaSndyclqQsJFrt7i6WkZDBu9HcYNfLbJCWlxux5pX3RUEkb0L17d1JTUyktLeX8+fPxLkcCvzkYueLRgb59v8SAfrfSK+MLVI3sxYa/LiYxMZkEi91byCyBnJyJDEj7GwqGVFB0Yh+HDulrAqU2BXecJCUlceONNzJz5ky+9a1vMXToUDZs2MD777/PCy+8QGFhYc2f5hIf1d82kpycRs6QSWR3HYmRQEnpfwAwqP8Yene6IXb7c+fUqYN061XAoG6TGHljIefOnaC4+FDM9iHtg4K7lWVkZHDHHXcwceJE7rjjDjIzL19lN336dG677TYeeeQRFi1axM6dO3n11VcV4HE2bNhkhg/+z6Ql9WBXwcvs3r0aALNEzGI52uicOJFH1077MR9Fv+6jGTJkPGfOFHDpUmUM9yNhp+BuJZmZmXzzm9/kxz/+McOGDSMhoe43vJnRp08fnnjiCcrKyti+fTsLFizgwIED7NypS6JbW58+X+BLOdPp0/lGjpXu4NNDGzh79hidO/dusX3u2b+ajKSzDOs5nS8P/RZnzx4nN/cvLbY/CZ96g9vM0oC/AKnB+v/m7k8ENwJeDvQEtgH3unu5maUCS4ExwCngLnfPb6H627SEhATGjBnD448/ztChQxkxYkSjtk9NTWXChAmsWLGCwsJCXnvtNX73u9/xySefcPr06RaqWqolJaUwbNhkBmZO4JJXUnD6Qw4d2gZAnz7XU+kXOXBqNX07jyIxIfJWSkvqQVJC8z5UPHEil/8ov0hm+nX07TyCG4b+DSdP5nHmzNFm90nah4YccZcBU9291MySgQ1mthr478DP3X25mf0SeABYFPw87e7Xm9ls4KfAXS1Uf5vUv39/7rvvPiZNmsS0adNIS2v+qV3Z2dnMnz+f+fPns2nTJn7xi1+wZcsW8vLyqKrSvZhbQq9eOQzMGkPnlL7sKXqND7b+gYsXSwA4dGgb588Xk5CQxNCht9AlNZ1eaal4ykBI7AxARnIW3dOGAJBgCaQkdvncJe8AZZXnOFX6KceP76tpq6i4yN5P1tK3z5cY0e87DOn9VU584QAfbF2mIRMBGnazYAdKg9nk4OHAVOC/Bu1LgCeJBPfMYBrg34DnzMy8nX+zfHp6OuPHj+fuu+9m2rRpDBky5KrDIU1V/aafNGkSEyZMoLy8nGXLlvH0009z7NgxHYXHUEZGL0aNmMWArhM4eX4fBw9v4MyZgprlVVXlNWFbWLiHvunpjOmZSb5155xFjri7d+9PVtYXAUhOTKdP18iFOobRJ+MmkhJSKas6x2flZ7hwofhz+z9//hRbP1pORlof+mR8idTULiQmpii4BWjgGLeZJRIZDrke+AXwKXDG3atfRUeB/sF0f+AIgLtXmlkJkeGUkzGsu00wM3r16sV3v/tdxowZw4wZM2JydN0QCQkJpKWlMWfOHO666y4OHTrEc889x5IlSygvL6eioqJV6oiVtLQ0Nm7cSEZGRhu5e8xFjp3YTueuXbhYdpa8g/9OWloKUPsbAAHOAu8UFwOXA7i4eD/5+ZGx6ZSUTgwYMBKIvG4GDxpHcnI6AGlpyWRkZFBVVUl+fj4lJSV06tSJM2cOs333y/ToMZBDh7dSUXGhJTssIWKNeZOYWXfgVeB/Ay+6+/VB+0BgtbvfZGa7gOnufjRY9ikwwd1PXvFc84B5weyYZvekFSUkJJCVlcWcOXN49NFH6dOnT8yPrpuioqKCU6dOsXbtWn71q1/x17/+tc2dkVL9e+ratSuzZs2qmf/+979PVlZWPEurxSyRlJR03J3y8tieX5+S0qnmjBT3S5SX1w7lV155hY8//pjExBSqqsp57bXXOHny8tuorf3bSuy5u9XV3qjgBjCzfwQ+A/4e6BscVU8CnnT328zszWD6r2aWBBwDel9rqCQ9Pd0vXmz7dwJJTExk2rRpPPbYYwwZMoTBgwfHu6SrKi4upqioiOeee44//elPFBUVxaWOTp068fWvf52EhATMjEcffZTs7GySk5PJycmpNeYrdXN3Dh06RPX75OLFiyxYsIALFy4H/rZt2zh6VB9gtidNDm4z6w1UuPsZM0sH3iLygeP9wIqoDyd3uPv/M7OHgS+7+/zgw8lvufud19rHzTff7IsWLaqZLy4u5umnn6750O3MmTPs3r27wZ2NtRtuuIEHH3yQW265hdGjR5OSUvefy23Vnj17yM/P56mnnmLTpk1UVsZ+nDQ1NZUxY8ZgZowePZrZs2fXtI8aNapN/EXS3u3fv7/miNzdWbhwIQUFl8flCwoKyM/Pj1N10hTNCe4RRD58TCTy3SYvu/s/m1kOkdMBM4GPgHvcvSw4ffB3wCgiA36z3T3vWvsYO3asb9169Ut7i4qK2LRpU818ZWUlzzzzDGfOnKlpi/WHc926dWPq1KnMmTOHcePGkZ2dXf9GbVxFRQVvv/02W7du5fnnn+fw4cONfo5+/frRrVvkS5VGjhxZE9CdOnVi6tSpCug2LDc3lz179tTML1u2jI8//rhmPi8vj7KysniUJlcRs6GSllBfcF/J3WuN723atOlzL8oXXniBgwcv3137s88+o6Sk5JrPm5iYSO/evZk8eTIPPfQQt956a7sMInenoKCA1atX88orr/Dee+/V+o6Ubt26kZ4e+fBsxIgRzJo1C4ApU6aQk5MDRD5ka4+/n47i0qVLn/sg+PXXX+fUqVMArFmzho0bNzbofSMtp10Fd0NcvHjxc+c37969m9WrV9fMf/bZZyxevJjy8nIgcp70I488wty5c0lNTQ3dcEhTlZWVsX79erZs2fK59hkzZjB8eOT0taSkJFJT9U11HUn1mUl79uxh1apV/PGPf+Tw4cOUlZW1yFCb1K3DBXd9Ll26xMmTJ2uO3JOTk8nMzNSHZSJ1KC4upry8nNWrV7N582YOHDjAu+++q4u/WpiCW0RipqSkhGPHjrF48WLy8vLYsWMHeXnX/ChLmkDBLSIt5uDBgxw5coQFCxZQUlLCjh07OHv2bLzLCj0Ft4i0mo0bN7Jv3z6effZZKisryc3N1dh4Eyi4RaRVVZ/9VVVVxYoVK/joo4/4/e9/T2VlJSdOnIh3eaGg4BaRuKqqqqKsrIzTp0/z4osv8v777/Pee+9RWVlJGK6cjgcFt4i0KefPn6e0tJT9+/fz0ksvsXbtWvLz82udX96RXS24dQccEYmLjIwMMjIyyMrK4tZbb6WgoIDS0lLWr1/PW2+9xbp16zh37ly8y2yTdMQtIm2Ou7Njxw5+9rOfsWrVqporOjuaqx1x63plEWlzzIybb76ZpUuXsnr1aubOnUvv3i13n8+wUXCLSJs2btw4nn/+edasWcNDDz1EZmZmvEuKOwW3iLR5CQkJjB49moULF/Luu+8yf/58OnXqFO+y4kbBLSKhkZSUxE033cTChQv54IMPmDt3bqvdLrAtUXCLSOgkJyczfPhwFi9ezLZt27j33nvp3LlzvMtqNQpuEQmtpKQkhg8fzpIlS9iwYQP33ntvhxgDV3CLSOhFn4WyZs0aXnrpJUaMGEFiYmK8S2sRCm4RaVfGjRvH7Nmz2bJlC0uXLmXEiBHt7sYo9Qa3maWZ2RYz+9jMdpvZPwXtL5rZQTPbHjxGBu1mZgvNLNfMdpjZ6JbuhIhINDMjNTWVu+++m82bN/PLX/6SUaNGkZycHO/SYqIhR9xlwFR3vxkYCUw3s4nBsh+7+8jgsT1o+wYwLHjMAxbVekYRkVZgZqSlpTF37lzeeecdFi1axPjx40N/r9R6q/eI0mA2OXhc6zr5mcDSYLtNQHczC/8t0kUk1Lp168YDDzzAqlWrWLJkCZMmTYp3SU3WoP92zCzRzLYDRcBad98cLPpJMBzyczOrvptsf+BI1OZHgzYRkbjr2bMn99xzD2+88QbLly9n3Lhx8S6pRo8ePZg8eTKTJ0++5umNDfp2QHevAkaaWXfgVTO7CXgcOAakAIuBvwf+uaEFmtk8IkMpDBo0qKGbiYjERGZmJnfddRczZsxg1apVPPXUU+zatSumN0DOyMggJyenVvv111/Pgw8+WKu9V69ejB8/HoCxY8de9Xkb9bWu7n7GzN4Bprv7M0FzmZm9APyPYL4AGBi12YCg7crnWkwk8Bk7dmz8v6JQRDqkLl26cOedd3L77bezYsUKFixYwN69eykvL6+1bmpqap1fdnXdddfxwAMP1Grv27cv06ZNq9VuZpjV+cV/DVJvcJtZb6AiCO10YBrwUzPLdvdCi+z9dmBXsMlK4AdmthyYAJS4e2GTKxQRaWHRZ6F8+9vfZtmyZRw7dqzWegMGDGDWrFm12hMSElr1lMOGHHFnA0vMLJHImPjL7v6Gma0PQt2A7cD8YP1VwAwgF7gAzI192SIisVcd4HPmzIl3KddUb3C7+w5gVB3tU6+yvgMPN780ERGpS7hPZhQR6YAU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMubu8a4BMzsH7It3HS2kF3Ay3kW0gPbaL2i/fVO/wmWwu/eua0FSa1dyFfvcfWy8i2gJZra1PfatvfYL2m/f1K/2Q0MlIiIho+AWEQmZthLci+NdQAtqr31rr/2C9ts39audaBMfToqISMO1lSNuERFpoLgHt5lNN7N9ZpZrZo/Fu57GMrPfmlmRme2Kass0s7VmdiD42SNoNzNbGPR1h5mNjl/l12ZmA83sHTPbY2a7zeyHQXuo+2ZmaWa2xcw+Dvr1T0H7dWa2Oaj/j2aWErSnBvO5wfIh8ay/PmaWaGYfmdkbwXx76Ve+me00s+1mtjVoC/VrsTniGtxmlgj8AvgGMBy428yGx7OmJngRmH5F22PAOncfBqwL5iHSz2HBYx6wqJVqbIpK4O/cfTgwEXg4+LcJe9/KgKnufjMwEphuZhOBnwI/d/frgdPAA8H6DwCng/afB+u1ZT8E9kbNt5d+AXzN3UdGnfoX9tdi07l73B7AJODNqPnHgcfjWVMT+zEE2BU1vw/IDqaziZynDvAr4O661mvrD+B1YFp76hvQCfgQmEDkAo6koL3mdQm8CUwKppOC9SzetV+lPwOIBNhU4A3A2kO/ghrzgV5XtLWb12JjH/EeKukPHImaPxq0hV2WuxcG08eArGA6lP0N/oweBWymHfQtGE7YDhQBa4FPgTPuXhmsEl17Tb+C5SVAz9atuMH+D/A/gUvBfE/aR78AHHjLzLaZ2bygLfSvxaZqK1dOtlvu7mYW2lN3zKwzsAL4kbufNbOaZWHtm7tXASPNrDvwKnBDnEtqNjP7T0CRu28zsynxrqcFfMXdC8ysD7DWzD6JXhjW12JTxfuIuwAYGDU/IGgLu+Nmlg0Q/CwK2kPVXzNLJhLaf3D3V4LmdtE3AHc/A7xDZAihu5lVH8hE117Tr2B5N+BUK5faELcA/8XM8oHlRIZL/i/h7xcA7l4Q/Cwi8p/teNrRa7Gx4h3cHwDDgk++U4DZwMo41xQLK4H7g+n7iYwPV7ffF3zqPREoifpTr02xyKH1b4C97v5s1KJQ983MegdH2phZOpFx+71EAnxWsNqV/aru7yxgvQcDp22Juz/u7gPcfQiR99F6d/8OIe8XgJllmFmX6mngb4FdhPy12CzxHmQHZgD7iYwz/q9419OE+pcBhUAFkbG0B4iMFa4DDgBvA5nBukbkLJpPgZ3A2HjXf41+fYXIuOIOYHvwmBH2vgEjgI+Cfu0C/jFozwG2ALnAn4DUoD0tmM8NlufEuw8N6OMU4I320q+gDx8Hj93VORH212JzHrpyUkQkZOI9VCIiIo2k4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZP4/aOAjA9p4Wi8AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F5paWqo7tWL2"
      },
      "source": [
        "## Policy Gradient\n",
        "Now, we can build a simple policy network. The network will return one of action in the action space."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J8tdmeD-tZew"
      },
      "source": [
        "class PolicyGradientNetwork(nn.Module):\n",
        "\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.fc1 = nn.Linear(8, 16)\n",
        "        self.fc2 = nn.Linear(16, 16)\n",
        "        self.fc3 = nn.Linear(16, 4)\n",
        "\n",
        "    def forward(self, state):\n",
        "        hid = torch.tanh(self.fc1(state))\n",
        "        hid = torch.tanh(self.fc2(hid))\n",
        "        return F.softmax(self.fc3(hid), dim=-1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ynbqJrhIFTC3"
      },
      "source": [
        "Then, we need to build a simple agent. The agent will acts according to the output of the policy network above. There are a few things can be done by agent:\n",
        "- `learn()`：update the policy network from log probabilities and rewards.\n",
        "- `sample()`：After receiving observation from the environment, utilize policy network to tell which action to take. The return values of this function includes action and log probabilities. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zZo-IxJx286z"
      },
      "source": [
        "from torch.optim.lr_scheduler import StepLR\n",
        "class PolicyGradientAgent():\n",
        "    \n",
        "    def __init__(self, network):\n",
        "        self.network = network\n",
        "        self.optimizer = optim.SGD(self.network.parameters(), lr=0.001)\n",
        "        \n",
        "    def forward(self, state):\n",
        "        return self.network(state)\n",
        "    def learn(self, log_probs, rewards):\n",
        "        loss = (-log_probs * rewards).sum() # You don't need to revise this to pass simple baseline (but you can)\n",
        "\n",
        "        self.optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        self.optimizer.step()\n",
        "        \n",
        "    def sample(self, state):\n",
        "        action_prob = self.network(torch.FloatTensor(state))\n",
        "        action_dist = Categorical(action_prob)\n",
        "        action = action_dist.sample()\n",
        "        log_prob = action_dist.log_prob(action)\n",
        "        return action.item(), log_prob"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ehPlnTKyRZf9"
      },
      "source": [
        "Lastly, build a network and agent to start training."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GfJIvML-RYjL"
      },
      "source": [
        "network = PolicyGradientNetwork()\n",
        "agent = PolicyGradientAgent(network)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ouv23glgf5Qt"
      },
      "source": [
        "## Trainin Agent\n",
        "\n",
        "Now let's start to train our agent.\n",
        "Through taking all the interactions between agent and environment as training data, the policy network can learn from all these attempts,"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "de3a153737af485ea436d7e8393d8248",
            "6345f8926212465291c04587353161f1",
            "3aa84c8c097d4858a0c38f18dec8b060",
            "6647e68cf064416ca593b990bed81edf",
            "a25de7edbbee47cc8094f43125efa39b",
            "a0bb5061ca0946ab89a89f9abadcebc8",
            "d99d8e6bcbe0445eb7eddbfe31277635",
            "5af1c6579f0f4c2eb03df4063749569e"
          ]
        },
        "id": "vg5rxBBaf38_",
        "outputId": "a1b06e39-99d6-4233-eda0-a3d58e77ffed"
      },
      "source": [
        "agent.network.train()  # Switch network into training mode \n",
        "EPISODE_PER_BATCH = 5  # update the  agent every 5 episode\n",
        "NUM_BATCH = 400        # totally update the agent for 400 time\n",
        "\n",
        "avg_total_rewards, avg_final_rewards = [], []\n",
        "\n",
        "prg_bar = tqdm(range(NUM_BATCH))\n",
        "for batch in prg_bar:\n",
        "\n",
        "    log_probs, rewards = [], []\n",
        "    total_rewards, final_rewards = [], []\n",
        "\n",
        "    # collect trajectory\n",
        "    for episode in range(EPISODE_PER_BATCH):\n",
        "        \n",
        "        state = env.reset()\n",
        "        total_reward, total_step = 0, 0\n",
        "        seq_rewards = []\n",
        "        while True:\n",
        "\n",
        "            action, log_prob = agent.sample(state) # at, log(at|st)\n",
        "            next_state, reward, done, _ = env.step(action)\n",
        "\n",
        "            log_probs.append(log_prob) # [log(a1|s1), log(a2|s2), ...., log(at|st)]\n",
        "            # seq_rewards.append(reward)\n",
        "            state = next_state\n",
        "            total_reward += reward\n",
        "            total_step += 1\n",
        "            rewards.append(reward) # change here\n",
        "            # ! IMPORTANT !\n",
        "            # Current reward implementation: immediate reward,  given action_list : a1, a2, a3 ......\n",
        "            #                                                         rewards :     r1, r2 ,r3 ......\n",
        "            # medium：change \"rewards\" to accumulative decaying reward, given action_list : a1,                           a2,                           a3, ......\n",
        "            #                                                           rewards :           r1+0.99*r2+0.99^2*r3+......, r2+0.99*r3+0.99^2*r4+...... ,  r3+0.99*r4+0.99^2*r5+ ......\n",
        "            # boss : implement DQN\n",
        "            if done:\n",
        "                final_rewards.append(reward)\n",
        "                total_rewards.append(total_reward)\n",
        "                \n",
        "                break\n",
        "\n",
        "    print(f\"rewards looks like \", np.shape(rewards))  \n",
        "    print(f\"log_probs looks like \", np.shape(log_probs))     \n",
        "    # record training process\n",
        "    avg_total_reward = sum(total_rewards) / len(total_rewards)\n",
        "    avg_final_reward = sum(final_rewards) / len(final_rewards)\n",
        "    avg_total_rewards.append(avg_total_reward)\n",
        "    avg_final_rewards.append(avg_final_reward)\n",
        "    prg_bar.set_description(f\"Total: {avg_total_reward: 4.1f}, Final: {avg_final_reward: 4.1f}\")\n",
        "\n",
        "    # update agent\n",
        "    # rewards = np.concatenate(rewards, axis=0)\n",
        "    rewards = (rewards - np.mean(rewards)) / (np.std(rewards) + 1e-9)  # normalize the reward \n",
        "    agent.learn(torch.stack(log_probs), torch.from_numpy(rewards))\n",
        "    print(\"logs prob looks like \", torch.stack(log_probs).size())\n",
        "    print(\"torch.from_numpy(rewards) looks like \", torch.from_numpy(rewards).size())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "de3a153737af485ea436d7e8393d8248",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=400.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "rewards looks like  (448,)\n",
            "log_probs looks like  (448,)\n",
            "logs prob looks like  torch.Size([448])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([448])\n",
            "rewards looks like  (515,)\n",
            "log_probs looks like  (515,)\n",
            "logs prob looks like  torch.Size([515])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([515])\n",
            "rewards looks like  (392,)\n",
            "log_probs looks like  (392,)\n",
            "logs prob looks like  torch.Size([392])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([392])\n",
            "rewards looks like  (518,)\n",
            "log_probs looks like  (518,)\n",
            "logs prob looks like  torch.Size([518])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([518])\n",
            "rewards looks like  (472,)\n",
            "log_probs looks like  (472,)\n",
            "logs prob looks like  torch.Size([472])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([472])\n",
            "rewards looks like  (530,)\n",
            "log_probs looks like  (530,)\n",
            "logs prob looks like  torch.Size([530])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([530])\n",
            "rewards looks like  (463,)\n",
            "log_probs looks like  (463,)\n",
            "logs prob looks like  torch.Size([463])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([463])\n",
            "rewards looks like  (540,)\n",
            "log_probs looks like  (540,)\n",
            "logs prob looks like  torch.Size([540])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([540])\n",
            "rewards looks like  (513,)\n",
            "log_probs looks like  (513,)\n",
            "logs prob looks like  torch.Size([513])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([513])\n",
            "rewards looks like  (449,)\n",
            "log_probs looks like  (449,)\n",
            "logs prob looks like  torch.Size([449])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([449])\n",
            "rewards looks like  (602,)\n",
            "log_probs looks like  (602,)\n",
            "logs prob looks like  torch.Size([602])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([602])\n",
            "rewards looks like  (542,)\n",
            "log_probs looks like  (542,)\n",
            "logs prob looks like  torch.Size([542])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([542])\n",
            "rewards looks like  (503,)\n",
            "log_probs looks like  (503,)\n",
            "logs prob looks like  torch.Size([503])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([503])\n",
            "rewards looks like  (470,)\n",
            "log_probs looks like  (470,)\n",
            "logs prob looks like  torch.Size([470])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([470])\n",
            "rewards looks like  (518,)\n",
            "log_probs looks like  (518,)\n",
            "logs prob looks like  torch.Size([518])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([518])\n",
            "rewards looks like  (421,)\n",
            "log_probs looks like  (421,)\n",
            "logs prob looks like  torch.Size([421])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([421])\n",
            "rewards looks like  (592,)\n",
            "log_probs looks like  (592,)\n",
            "logs prob looks like  torch.Size([592])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([592])\n",
            "rewards looks like  (520,)\n",
            "log_probs looks like  (520,)\n",
            "logs prob looks like  torch.Size([520])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([520])\n",
            "rewards looks like  (494,)\n",
            "log_probs looks like  (494,)\n",
            "logs prob looks like  torch.Size([494])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([494])\n",
            "rewards looks like  (461,)\n",
            "log_probs looks like  (461,)\n",
            "logs prob looks like  torch.Size([461])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([461])\n",
            "rewards looks like  (572,)\n",
            "log_probs looks like  (572,)\n",
            "logs prob looks like  torch.Size([572])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([572])\n",
            "rewards looks like  (593,)\n",
            "log_probs looks like  (593,)\n",
            "logs prob looks like  torch.Size([593])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([593])\n",
            "rewards looks like  (569,)\n",
            "log_probs looks like  (569,)\n",
            "logs prob looks like  torch.Size([569])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([569])\n",
            "rewards looks like  (546,)\n",
            "log_probs looks like  (546,)\n",
            "logs prob looks like  torch.Size([546])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([546])\n",
            "rewards looks like  (612,)\n",
            "log_probs looks like  (612,)\n",
            "logs prob looks like  torch.Size([612])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([612])\n",
            "rewards looks like  (534,)\n",
            "log_probs looks like  (534,)\n",
            "logs prob looks like  torch.Size([534])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([534])\n",
            "rewards looks like  (513,)\n",
            "log_probs looks like  (513,)\n",
            "logs prob looks like  torch.Size([513])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([513])\n",
            "rewards looks like  (513,)\n",
            "log_probs looks like  (513,)\n",
            "logs prob looks like  torch.Size([513])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([513])\n",
            "rewards looks like  (535,)\n",
            "log_probs looks like  (535,)\n",
            "logs prob looks like  torch.Size([535])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([535])\n",
            "rewards looks like  (533,)\n",
            "log_probs looks like  (533,)\n",
            "logs prob looks like  torch.Size([533])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([533])\n",
            "rewards looks like  (521,)\n",
            "log_probs looks like  (521,)\n",
            "logs prob looks like  torch.Size([521])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([521])\n",
            "rewards looks like  (566,)\n",
            "log_probs looks like  (566,)\n",
            "logs prob looks like  torch.Size([566])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([566])\n",
            "rewards looks like  (586,)\n",
            "log_probs looks like  (586,)\n",
            "logs prob looks like  torch.Size([586])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([586])\n",
            "rewards looks like  (575,)\n",
            "log_probs looks like  (575,)\n",
            "logs prob looks like  torch.Size([575])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([575])\n",
            "rewards looks like  (709,)\n",
            "log_probs looks like  (709,)\n",
            "logs prob looks like  torch.Size([709])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([709])\n",
            "rewards looks like  (486,)\n",
            "log_probs looks like  (486,)\n",
            "logs prob looks like  torch.Size([486])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([486])\n",
            "rewards looks like  (557,)\n",
            "log_probs looks like  (557,)\n",
            "logs prob looks like  torch.Size([557])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([557])\n",
            "rewards looks like  (517,)\n",
            "log_probs looks like  (517,)\n",
            "logs prob looks like  torch.Size([517])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([517])\n",
            "rewards looks like  (550,)\n",
            "log_probs looks like  (550,)\n",
            "logs prob looks like  torch.Size([550])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([550])\n",
            "rewards looks like  (690,)\n",
            "log_probs looks like  (690,)\n",
            "logs prob looks like  torch.Size([690])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([690])\n",
            "rewards looks like  (591,)\n",
            "log_probs looks like  (591,)\n",
            "logs prob looks like  torch.Size([591])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([591])\n",
            "rewards looks like  (689,)\n",
            "log_probs looks like  (689,)\n",
            "logs prob looks like  torch.Size([689])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([689])\n",
            "rewards looks like  (1059,)\n",
            "log_probs looks like  (1059,)\n",
            "logs prob looks like  torch.Size([1059])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1059])\n",
            "rewards looks like  (619,)\n",
            "log_probs looks like  (619,)\n",
            "logs prob looks like  torch.Size([619])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([619])\n",
            "rewards looks like  (527,)\n",
            "log_probs looks like  (527,)\n",
            "logs prob looks like  torch.Size([527])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([527])\n",
            "rewards looks like  (514,)\n",
            "log_probs looks like  (514,)\n",
            "logs prob looks like  torch.Size([514])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([514])\n",
            "rewards looks like  (655,)\n",
            "log_probs looks like  (655,)\n",
            "logs prob looks like  torch.Size([655])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([655])\n",
            "rewards looks like  (667,)\n",
            "log_probs looks like  (667,)\n",
            "logs prob looks like  torch.Size([667])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([667])\n",
            "rewards looks like  (712,)\n",
            "log_probs looks like  (712,)\n",
            "logs prob looks like  torch.Size([712])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([712])\n",
            "rewards looks like  (636,)\n",
            "log_probs looks like  (636,)\n",
            "logs prob looks like  torch.Size([636])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([636])\n",
            "rewards looks like  (620,)\n",
            "log_probs looks like  (620,)\n",
            "logs prob looks like  torch.Size([620])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([620])\n",
            "rewards looks like  (543,)\n",
            "log_probs looks like  (543,)\n",
            "logs prob looks like  torch.Size([543])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([543])\n",
            "rewards looks like  (586,)\n",
            "log_probs looks like  (586,)\n",
            "logs prob looks like  torch.Size([586])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([586])\n",
            "rewards looks like  (498,)\n",
            "log_probs looks like  (498,)\n",
            "logs prob looks like  torch.Size([498])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([498])\n",
            "rewards looks like  (586,)\n",
            "log_probs looks like  (586,)\n",
            "logs prob looks like  torch.Size([586])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([586])\n",
            "rewards looks like  (591,)\n",
            "log_probs looks like  (591,)\n",
            "logs prob looks like  torch.Size([591])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([591])\n",
            "rewards looks like  (693,)\n",
            "log_probs looks like  (693,)\n",
            "logs prob looks like  torch.Size([693])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([693])\n",
            "rewards looks like  (648,)\n",
            "log_probs looks like  (648,)\n",
            "logs prob looks like  torch.Size([648])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([648])\n",
            "rewards looks like  (513,)\n",
            "log_probs looks like  (513,)\n",
            "logs prob looks like  torch.Size([513])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([513])\n",
            "rewards looks like  (574,)\n",
            "log_probs looks like  (574,)\n",
            "logs prob looks like  torch.Size([574])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([574])\n",
            "rewards looks like  (718,)\n",
            "log_probs looks like  (718,)\n",
            "logs prob looks like  torch.Size([718])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([718])\n",
            "rewards looks like  (730,)\n",
            "log_probs looks like  (730,)\n",
            "logs prob looks like  torch.Size([730])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([730])\n",
            "rewards looks like  (668,)\n",
            "log_probs looks like  (668,)\n",
            "logs prob looks like  torch.Size([668])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([668])\n",
            "rewards looks like  (754,)\n",
            "log_probs looks like  (754,)\n",
            "logs prob looks like  torch.Size([754])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([754])\n",
            "rewards looks like  (712,)\n",
            "log_probs looks like  (712,)\n",
            "logs prob looks like  torch.Size([712])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([712])\n",
            "rewards looks like  (470,)\n",
            "log_probs looks like  (470,)\n",
            "logs prob looks like  torch.Size([470])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([470])\n",
            "rewards looks like  (665,)\n",
            "log_probs looks like  (665,)\n",
            "logs prob looks like  torch.Size([665])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([665])\n",
            "rewards looks like  (585,)\n",
            "log_probs looks like  (585,)\n",
            "logs prob looks like  torch.Size([585])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([585])\n",
            "rewards looks like  (512,)\n",
            "log_probs looks like  (512,)\n",
            "logs prob looks like  torch.Size([512])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([512])\n",
            "rewards looks like  (702,)\n",
            "log_probs looks like  (702,)\n",
            "logs prob looks like  torch.Size([702])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([702])\n",
            "rewards looks like  (596,)\n",
            "log_probs looks like  (596,)\n",
            "logs prob looks like  torch.Size([596])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([596])\n",
            "rewards looks like  (626,)\n",
            "log_probs looks like  (626,)\n",
            "logs prob looks like  torch.Size([626])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([626])\n",
            "rewards looks like  (566,)\n",
            "log_probs looks like  (566,)\n",
            "logs prob looks like  torch.Size([566])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([566])\n",
            "rewards looks like  (717,)\n",
            "log_probs looks like  (717,)\n",
            "logs prob looks like  torch.Size([717])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([717])\n",
            "rewards looks like  (708,)\n",
            "log_probs looks like  (708,)\n",
            "logs prob looks like  torch.Size([708])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([708])\n",
            "rewards looks like  (565,)\n",
            "log_probs looks like  (565,)\n",
            "logs prob looks like  torch.Size([565])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([565])\n",
            "rewards looks like  (450,)\n",
            "log_probs looks like  (450,)\n",
            "logs prob looks like  torch.Size([450])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([450])\n",
            "rewards looks like  (584,)\n",
            "log_probs looks like  (584,)\n",
            "logs prob looks like  torch.Size([584])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([584])\n",
            "rewards looks like  (670,)\n",
            "log_probs looks like  (670,)\n",
            "logs prob looks like  torch.Size([670])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([670])\n",
            "rewards looks like  (691,)\n",
            "log_probs looks like  (691,)\n",
            "logs prob looks like  torch.Size([691])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([691])\n",
            "rewards looks like  (760,)\n",
            "log_probs looks like  (760,)\n",
            "logs prob looks like  torch.Size([760])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([760])\n",
            "rewards looks like  (752,)\n",
            "log_probs looks like  (752,)\n",
            "logs prob looks like  torch.Size([752])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([752])\n",
            "rewards looks like  (478,)\n",
            "log_probs looks like  (478,)\n",
            "logs prob looks like  torch.Size([478])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([478])\n",
            "rewards looks like  (553,)\n",
            "log_probs looks like  (553,)\n",
            "logs prob looks like  torch.Size([553])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([553])\n",
            "rewards looks like  (1660,)\n",
            "log_probs looks like  (1660,)\n",
            "logs prob looks like  torch.Size([1660])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1660])\n",
            "rewards looks like  (751,)\n",
            "log_probs looks like  (751,)\n",
            "logs prob looks like  torch.Size([751])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([751])\n",
            "rewards looks like  (801,)\n",
            "log_probs looks like  (801,)\n",
            "logs prob looks like  torch.Size([801])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([801])\n",
            "rewards looks like  (715,)\n",
            "log_probs looks like  (715,)\n",
            "logs prob looks like  torch.Size([715])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([715])\n",
            "rewards looks like  (708,)\n",
            "log_probs looks like  (708,)\n",
            "logs prob looks like  torch.Size([708])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([708])\n",
            "rewards looks like  (609,)\n",
            "log_probs looks like  (609,)\n",
            "logs prob looks like  torch.Size([609])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([609])\n",
            "rewards looks like  (732,)\n",
            "log_probs looks like  (732,)\n",
            "logs prob looks like  torch.Size([732])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([732])\n",
            "rewards looks like  (603,)\n",
            "log_probs looks like  (603,)\n",
            "logs prob looks like  torch.Size([603])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([603])\n",
            "rewards looks like  (603,)\n",
            "log_probs looks like  (603,)\n",
            "logs prob looks like  torch.Size([603])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([603])\n",
            "rewards looks like  (665,)\n",
            "log_probs looks like  (665,)\n",
            "logs prob looks like  torch.Size([665])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([665])\n",
            "rewards looks like  (658,)\n",
            "log_probs looks like  (658,)\n",
            "logs prob looks like  torch.Size([658])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([658])\n",
            "rewards looks like  (783,)\n",
            "log_probs looks like  (783,)\n",
            "logs prob looks like  torch.Size([783])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([783])\n",
            "rewards looks like  (652,)\n",
            "log_probs looks like  (652,)\n",
            "logs prob looks like  torch.Size([652])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([652])\n",
            "rewards looks like  (892,)\n",
            "log_probs looks like  (892,)\n",
            "logs prob looks like  torch.Size([892])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([892])\n",
            "rewards looks like  (821,)\n",
            "log_probs looks like  (821,)\n",
            "logs prob looks like  torch.Size([821])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([821])\n",
            "rewards looks like  (986,)\n",
            "log_probs looks like  (986,)\n",
            "logs prob looks like  torch.Size([986])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([986])\n",
            "rewards looks like  (916,)\n",
            "log_probs looks like  (916,)\n",
            "logs prob looks like  torch.Size([916])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([916])\n",
            "rewards looks like  (742,)\n",
            "log_probs looks like  (742,)\n",
            "logs prob looks like  torch.Size([742])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([742])\n",
            "rewards looks like  (604,)\n",
            "log_probs looks like  (604,)\n",
            "logs prob looks like  torch.Size([604])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([604])\n",
            "rewards looks like  (818,)\n",
            "log_probs looks like  (818,)\n",
            "logs prob looks like  torch.Size([818])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([818])\n",
            "rewards looks like  (855,)\n",
            "log_probs looks like  (855,)\n",
            "logs prob looks like  torch.Size([855])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([855])\n",
            "rewards looks like  (795,)\n",
            "log_probs looks like  (795,)\n",
            "logs prob looks like  torch.Size([795])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([795])\n",
            "rewards looks like  (868,)\n",
            "log_probs looks like  (868,)\n",
            "logs prob looks like  torch.Size([868])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([868])\n",
            "rewards looks like  (800,)\n",
            "log_probs looks like  (800,)\n",
            "logs prob looks like  torch.Size([800])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([800])\n",
            "rewards looks like  (820,)\n",
            "log_probs looks like  (820,)\n",
            "logs prob looks like  torch.Size([820])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([820])\n",
            "rewards looks like  (760,)\n",
            "log_probs looks like  (760,)\n",
            "logs prob looks like  torch.Size([760])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([760])\n",
            "rewards looks like  (886,)\n",
            "log_probs looks like  (886,)\n",
            "logs prob looks like  torch.Size([886])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([886])\n",
            "rewards looks like  (1027,)\n",
            "log_probs looks like  (1027,)\n",
            "logs prob looks like  torch.Size([1027])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1027])\n",
            "rewards looks like  (819,)\n",
            "log_probs looks like  (819,)\n",
            "logs prob looks like  torch.Size([819])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([819])\n",
            "rewards looks like  (934,)\n",
            "log_probs looks like  (934,)\n",
            "logs prob looks like  torch.Size([934])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([934])\n",
            "rewards looks like  (1648,)\n",
            "log_probs looks like  (1648,)\n",
            "logs prob looks like  torch.Size([1648])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1648])\n",
            "rewards looks like  (1057,)\n",
            "log_probs looks like  (1057,)\n",
            "logs prob looks like  torch.Size([1057])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1057])\n",
            "rewards looks like  (861,)\n",
            "log_probs looks like  (861,)\n",
            "logs prob looks like  torch.Size([861])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([861])\n",
            "rewards looks like  (1533,)\n",
            "log_probs looks like  (1533,)\n",
            "logs prob looks like  torch.Size([1533])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1533])\n",
            "rewards looks like  (920,)\n",
            "log_probs looks like  (920,)\n",
            "logs prob looks like  torch.Size([920])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([920])\n",
            "rewards looks like  (905,)\n",
            "log_probs looks like  (905,)\n",
            "logs prob looks like  torch.Size([905])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([905])\n",
            "rewards looks like  (814,)\n",
            "log_probs looks like  (814,)\n",
            "logs prob looks like  torch.Size([814])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([814])\n",
            "rewards looks like  (809,)\n",
            "log_probs looks like  (809,)\n",
            "logs prob looks like  torch.Size([809])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([809])\n",
            "rewards looks like  (873,)\n",
            "log_probs looks like  (873,)\n",
            "logs prob looks like  torch.Size([873])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([873])\n",
            "rewards looks like  (727,)\n",
            "log_probs looks like  (727,)\n",
            "logs prob looks like  torch.Size([727])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([727])\n",
            "rewards looks like  (1129,)\n",
            "log_probs looks like  (1129,)\n",
            "logs prob looks like  torch.Size([1129])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1129])\n",
            "rewards looks like  (1394,)\n",
            "log_probs looks like  (1394,)\n",
            "logs prob looks like  torch.Size([1394])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1394])\n",
            "rewards looks like  (884,)\n",
            "log_probs looks like  (884,)\n",
            "logs prob looks like  torch.Size([884])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([884])\n",
            "rewards looks like  (1132,)\n",
            "log_probs looks like  (1132,)\n",
            "logs prob looks like  torch.Size([1132])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1132])\n",
            "rewards looks like  (1007,)\n",
            "log_probs looks like  (1007,)\n",
            "logs prob looks like  torch.Size([1007])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1007])\n",
            "rewards looks like  (711,)\n",
            "log_probs looks like  (711,)\n",
            "logs prob looks like  torch.Size([711])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([711])\n",
            "rewards looks like  (836,)\n",
            "log_probs looks like  (836,)\n",
            "logs prob looks like  torch.Size([836])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([836])\n",
            "rewards looks like  (1514,)\n",
            "log_probs looks like  (1514,)\n",
            "logs prob looks like  torch.Size([1514])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1514])\n",
            "rewards looks like  (896,)\n",
            "log_probs looks like  (896,)\n",
            "logs prob looks like  torch.Size([896])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([896])\n",
            "rewards looks like  (912,)\n",
            "log_probs looks like  (912,)\n",
            "logs prob looks like  torch.Size([912])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([912])\n",
            "rewards looks like  (1478,)\n",
            "log_probs looks like  (1478,)\n",
            "logs prob looks like  torch.Size([1478])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1478])\n",
            "rewards looks like  (1279,)\n",
            "log_probs looks like  (1279,)\n",
            "logs prob looks like  torch.Size([1279])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1279])\n",
            "rewards looks like  (676,)\n",
            "log_probs looks like  (676,)\n",
            "logs prob looks like  torch.Size([676])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([676])\n",
            "rewards looks like  (1768,)\n",
            "log_probs looks like  (1768,)\n",
            "logs prob looks like  torch.Size([1768])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1768])\n",
            "rewards looks like  (897,)\n",
            "log_probs looks like  (897,)\n",
            "logs prob looks like  torch.Size([897])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([897])\n",
            "rewards looks like  (1119,)\n",
            "log_probs looks like  (1119,)\n",
            "logs prob looks like  torch.Size([1119])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1119])\n",
            "rewards looks like  (943,)\n",
            "log_probs looks like  (943,)\n",
            "logs prob looks like  torch.Size([943])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([943])\n",
            "rewards looks like  (1255,)\n",
            "log_probs looks like  (1255,)\n",
            "logs prob looks like  torch.Size([1255])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1255])\n",
            "rewards looks like  (861,)\n",
            "log_probs looks like  (861,)\n",
            "logs prob looks like  torch.Size([861])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([861])\n",
            "rewards looks like  (1149,)\n",
            "log_probs looks like  (1149,)\n",
            "logs prob looks like  torch.Size([1149])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1149])\n",
            "rewards looks like  (1229,)\n",
            "log_probs looks like  (1229,)\n",
            "logs prob looks like  torch.Size([1229])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1229])\n",
            "rewards looks like  (1680,)\n",
            "log_probs looks like  (1680,)\n",
            "logs prob looks like  torch.Size([1680])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1680])\n",
            "rewards looks like  (1731,)\n",
            "log_probs looks like  (1731,)\n",
            "logs prob looks like  torch.Size([1731])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1731])\n",
            "rewards looks like  (1017,)\n",
            "log_probs looks like  (1017,)\n",
            "logs prob looks like  torch.Size([1017])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1017])\n",
            "rewards looks like  (990,)\n",
            "log_probs looks like  (990,)\n",
            "logs prob looks like  torch.Size([990])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([990])\n",
            "rewards looks like  (1020,)\n",
            "log_probs looks like  (1020,)\n",
            "logs prob looks like  torch.Size([1020])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1020])\n",
            "rewards looks like  (1240,)\n",
            "log_probs looks like  (1240,)\n",
            "logs prob looks like  torch.Size([1240])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1240])\n",
            "rewards looks like  (774,)\n",
            "log_probs looks like  (774,)\n",
            "logs prob looks like  torch.Size([774])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([774])\n",
            "rewards looks like  (1069,)\n",
            "log_probs looks like  (1069,)\n",
            "logs prob looks like  torch.Size([1069])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1069])\n",
            "rewards looks like  (1355,)\n",
            "log_probs looks like  (1355,)\n",
            "logs prob looks like  torch.Size([1355])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1355])\n",
            "rewards looks like  (1556,)\n",
            "log_probs looks like  (1556,)\n",
            "logs prob looks like  torch.Size([1556])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1556])\n",
            "rewards looks like  (1840,)\n",
            "log_probs looks like  (1840,)\n",
            "logs prob looks like  torch.Size([1840])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1840])\n",
            "rewards looks like  (1352,)\n",
            "log_probs looks like  (1352,)\n",
            "logs prob looks like  torch.Size([1352])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1352])\n",
            "rewards looks like  (1617,)\n",
            "log_probs looks like  (1617,)\n",
            "logs prob looks like  torch.Size([1617])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1617])\n",
            "rewards looks like  (1637,)\n",
            "log_probs looks like  (1637,)\n",
            "logs prob looks like  torch.Size([1637])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1637])\n",
            "rewards looks like  (1606,)\n",
            "log_probs looks like  (1606,)\n",
            "logs prob looks like  torch.Size([1606])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1606])\n",
            "rewards looks like  (860,)\n",
            "log_probs looks like  (860,)\n",
            "logs prob looks like  torch.Size([860])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([860])\n",
            "rewards looks like  (1780,)\n",
            "log_probs looks like  (1780,)\n",
            "logs prob looks like  torch.Size([1780])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1780])\n",
            "rewards looks like  (2248,)\n",
            "log_probs looks like  (2248,)\n",
            "logs prob looks like  torch.Size([2248])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2248])\n",
            "rewards looks like  (1410,)\n",
            "log_probs looks like  (1410,)\n",
            "logs prob looks like  torch.Size([1410])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1410])\n",
            "rewards looks like  (557,)\n",
            "log_probs looks like  (557,)\n",
            "logs prob looks like  torch.Size([557])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([557])\n",
            "rewards looks like  (719,)\n",
            "log_probs looks like  (719,)\n",
            "logs prob looks like  torch.Size([719])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([719])\n",
            "rewards looks like  (1919,)\n",
            "log_probs looks like  (1919,)\n",
            "logs prob looks like  torch.Size([1919])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1919])\n",
            "rewards looks like  (1250,)\n",
            "log_probs looks like  (1250,)\n",
            "logs prob looks like  torch.Size([1250])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1250])\n",
            "rewards looks like  (1054,)\n",
            "log_probs looks like  (1054,)\n",
            "logs prob looks like  torch.Size([1054])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1054])\n",
            "rewards looks like  (1276,)\n",
            "log_probs looks like  (1276,)\n",
            "logs prob looks like  torch.Size([1276])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1276])\n",
            "rewards looks like  (1040,)\n",
            "log_probs looks like  (1040,)\n",
            "logs prob looks like  torch.Size([1040])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1040])\n",
            "rewards looks like  (991,)\n",
            "log_probs looks like  (991,)\n",
            "logs prob looks like  torch.Size([991])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([991])\n",
            "rewards looks like  (1390,)\n",
            "log_probs looks like  (1390,)\n",
            "logs prob looks like  torch.Size([1390])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1390])\n",
            "rewards looks like  (1349,)\n",
            "log_probs looks like  (1349,)\n",
            "logs prob looks like  torch.Size([1349])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1349])\n",
            "rewards looks like  (1332,)\n",
            "log_probs looks like  (1332,)\n",
            "logs prob looks like  torch.Size([1332])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1332])\n",
            "rewards looks like  (1378,)\n",
            "log_probs looks like  (1378,)\n",
            "logs prob looks like  torch.Size([1378])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1378])\n",
            "rewards looks like  (1967,)\n",
            "log_probs looks like  (1967,)\n",
            "logs prob looks like  torch.Size([1967])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1967])\n",
            "rewards looks like  (1789,)\n",
            "log_probs looks like  (1789,)\n",
            "logs prob looks like  torch.Size([1789])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1789])\n",
            "rewards looks like  (1325,)\n",
            "log_probs looks like  (1325,)\n",
            "logs prob looks like  torch.Size([1325])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1325])\n",
            "rewards looks like  (1685,)\n",
            "log_probs looks like  (1685,)\n",
            "logs prob looks like  torch.Size([1685])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1685])\n",
            "rewards looks like  (1895,)\n",
            "log_probs looks like  (1895,)\n",
            "logs prob looks like  torch.Size([1895])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1895])\n",
            "rewards looks like  (1920,)\n",
            "log_probs looks like  (1920,)\n",
            "logs prob looks like  torch.Size([1920])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1920])\n",
            "rewards looks like  (1522,)\n",
            "log_probs looks like  (1522,)\n",
            "logs prob looks like  torch.Size([1522])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1522])\n",
            "rewards looks like  (1173,)\n",
            "log_probs looks like  (1173,)\n",
            "logs prob looks like  torch.Size([1173])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1173])\n",
            "rewards looks like  (2136,)\n",
            "log_probs looks like  (2136,)\n",
            "logs prob looks like  torch.Size([2136])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2136])\n",
            "rewards looks like  (1696,)\n",
            "log_probs looks like  (1696,)\n",
            "logs prob looks like  torch.Size([1696])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1696])\n",
            "rewards looks like  (568,)\n",
            "log_probs looks like  (568,)\n",
            "logs prob looks like  torch.Size([568])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([568])\n",
            "rewards looks like  (1475,)\n",
            "log_probs looks like  (1475,)\n",
            "logs prob looks like  torch.Size([1475])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1475])\n",
            "rewards looks like  (2470,)\n",
            "log_probs looks like  (2470,)\n",
            "logs prob looks like  torch.Size([2470])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2470])\n",
            "rewards looks like  (3053,)\n",
            "log_probs looks like  (3053,)\n",
            "logs prob looks like  torch.Size([3053])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([3053])\n",
            "rewards looks like  (915,)\n",
            "log_probs looks like  (915,)\n",
            "logs prob looks like  torch.Size([915])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([915])\n",
            "rewards looks like  (2049,)\n",
            "log_probs looks like  (2049,)\n",
            "logs prob looks like  torch.Size([2049])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2049])\n",
            "rewards looks like  (2068,)\n",
            "log_probs looks like  (2068,)\n",
            "logs prob looks like  torch.Size([2068])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2068])\n",
            "rewards looks like  (2528,)\n",
            "log_probs looks like  (2528,)\n",
            "logs prob looks like  torch.Size([2528])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2528])\n",
            "rewards looks like  (1839,)\n",
            "log_probs looks like  (1839,)\n",
            "logs prob looks like  torch.Size([1839])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1839])\n",
            "rewards looks like  (497,)\n",
            "log_probs looks like  (497,)\n",
            "logs prob looks like  torch.Size([497])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([497])\n",
            "rewards looks like  (627,)\n",
            "log_probs looks like  (627,)\n",
            "logs prob looks like  torch.Size([627])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([627])\n",
            "rewards looks like  (2354,)\n",
            "log_probs looks like  (2354,)\n",
            "logs prob looks like  torch.Size([2354])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2354])\n",
            "rewards looks like  (2394,)\n",
            "log_probs looks like  (2394,)\n",
            "logs prob looks like  torch.Size([2394])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2394])\n",
            "rewards looks like  (743,)\n",
            "log_probs looks like  (743,)\n",
            "logs prob looks like  torch.Size([743])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([743])\n",
            "rewards looks like  (1572,)\n",
            "log_probs looks like  (1572,)\n",
            "logs prob looks like  torch.Size([1572])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1572])\n",
            "rewards looks like  (2575,)\n",
            "log_probs looks like  (2575,)\n",
            "logs prob looks like  torch.Size([2575])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2575])\n",
            "rewards looks like  (2226,)\n",
            "log_probs looks like  (2226,)\n",
            "logs prob looks like  torch.Size([2226])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2226])\n",
            "rewards looks like  (541,)\n",
            "log_probs looks like  (541,)\n",
            "logs prob looks like  torch.Size([541])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([541])\n",
            "rewards looks like  (820,)\n",
            "log_probs looks like  (820,)\n",
            "logs prob looks like  torch.Size([820])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([820])\n",
            "rewards looks like  (2584,)\n",
            "log_probs looks like  (2584,)\n",
            "logs prob looks like  torch.Size([2584])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2584])\n",
            "rewards looks like  (1792,)\n",
            "log_probs looks like  (1792,)\n",
            "logs prob looks like  torch.Size([1792])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1792])\n",
            "rewards looks like  (1613,)\n",
            "log_probs looks like  (1613,)\n",
            "logs prob looks like  torch.Size([1613])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1613])\n",
            "rewards looks like  (4300,)\n",
            "log_probs looks like  (4300,)\n",
            "logs prob looks like  torch.Size([4300])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([4300])\n",
            "rewards looks like  (1602,)\n",
            "log_probs looks like  (1602,)\n",
            "logs prob looks like  torch.Size([1602])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1602])\n",
            "rewards looks like  (3313,)\n",
            "log_probs looks like  (3313,)\n",
            "logs prob looks like  torch.Size([3313])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([3313])\n",
            "rewards looks like  (1538,)\n",
            "log_probs looks like  (1538,)\n",
            "logs prob looks like  torch.Size([1538])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1538])\n",
            "rewards looks like  (1824,)\n",
            "log_probs looks like  (1824,)\n",
            "logs prob looks like  torch.Size([1824])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1824])\n",
            "rewards looks like  (1320,)\n",
            "log_probs looks like  (1320,)\n",
            "logs prob looks like  torch.Size([1320])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1320])\n",
            "rewards looks like  (2077,)\n",
            "log_probs looks like  (2077,)\n",
            "logs prob looks like  torch.Size([2077])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2077])\n",
            "rewards looks like  (1995,)\n",
            "log_probs looks like  (1995,)\n",
            "logs prob looks like  torch.Size([1995])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1995])\n",
            "rewards looks like  (1089,)\n",
            "log_probs looks like  (1089,)\n",
            "logs prob looks like  torch.Size([1089])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1089])\n",
            "rewards looks like  (1135,)\n",
            "log_probs looks like  (1135,)\n",
            "logs prob looks like  torch.Size([1135])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1135])\n",
            "rewards looks like  (1617,)\n",
            "log_probs looks like  (1617,)\n",
            "logs prob looks like  torch.Size([1617])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1617])\n",
            "rewards looks like  (942,)\n",
            "log_probs looks like  (942,)\n",
            "logs prob looks like  torch.Size([942])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([942])\n",
            "rewards looks like  (2006,)\n",
            "log_probs looks like  (2006,)\n",
            "logs prob looks like  torch.Size([2006])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2006])\n",
            "rewards looks like  (2204,)\n",
            "log_probs looks like  (2204,)\n",
            "logs prob looks like  torch.Size([2204])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2204])\n",
            "rewards looks like  (1060,)\n",
            "log_probs looks like  (1060,)\n",
            "logs prob looks like  torch.Size([1060])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1060])\n",
            "rewards looks like  (1994,)\n",
            "log_probs looks like  (1994,)\n",
            "logs prob looks like  torch.Size([1994])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1994])\n",
            "rewards looks like  (1118,)\n",
            "log_probs looks like  (1118,)\n",
            "logs prob looks like  torch.Size([1118])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1118])\n",
            "rewards looks like  (1298,)\n",
            "log_probs looks like  (1298,)\n",
            "logs prob looks like  torch.Size([1298])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1298])\n",
            "rewards looks like  (1377,)\n",
            "log_probs looks like  (1377,)\n",
            "logs prob looks like  torch.Size([1377])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1377])\n",
            "rewards looks like  (1902,)\n",
            "log_probs looks like  (1902,)\n",
            "logs prob looks like  torch.Size([1902])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1902])\n",
            "rewards looks like  (1982,)\n",
            "log_probs looks like  (1982,)\n",
            "logs prob looks like  torch.Size([1982])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1982])\n",
            "rewards looks like  (1625,)\n",
            "log_probs looks like  (1625,)\n",
            "logs prob looks like  torch.Size([1625])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1625])\n",
            "rewards looks like  (1947,)\n",
            "log_probs looks like  (1947,)\n",
            "logs prob looks like  torch.Size([1947])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1947])\n",
            "rewards looks like  (1589,)\n",
            "log_probs looks like  (1589,)\n",
            "logs prob looks like  torch.Size([1589])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1589])\n",
            "rewards looks like  (1625,)\n",
            "log_probs looks like  (1625,)\n",
            "logs prob looks like  torch.Size([1625])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1625])\n",
            "rewards looks like  (1492,)\n",
            "log_probs looks like  (1492,)\n",
            "logs prob looks like  torch.Size([1492])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1492])\n",
            "rewards looks like  (1347,)\n",
            "log_probs looks like  (1347,)\n",
            "logs prob looks like  torch.Size([1347])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1347])\n",
            "rewards looks like  (2110,)\n",
            "log_probs looks like  (2110,)\n",
            "logs prob looks like  torch.Size([2110])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2110])\n",
            "rewards looks like  (877,)\n",
            "log_probs looks like  (877,)\n",
            "logs prob looks like  torch.Size([877])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([877])\n",
            "rewards looks like  (1078,)\n",
            "log_probs looks like  (1078,)\n",
            "logs prob looks like  torch.Size([1078])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1078])\n",
            "rewards looks like  (2001,)\n",
            "log_probs looks like  (2001,)\n",
            "logs prob looks like  torch.Size([2001])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2001])\n",
            "rewards looks like  (1452,)\n",
            "log_probs looks like  (1452,)\n",
            "logs prob looks like  torch.Size([1452])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1452])\n",
            "rewards looks like  (1169,)\n",
            "log_probs looks like  (1169,)\n",
            "logs prob looks like  torch.Size([1169])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1169])\n",
            "rewards looks like  (1977,)\n",
            "log_probs looks like  (1977,)\n",
            "logs prob looks like  torch.Size([1977])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1977])\n",
            "rewards looks like  (1263,)\n",
            "log_probs looks like  (1263,)\n",
            "logs prob looks like  torch.Size([1263])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1263])\n",
            "rewards looks like  (2219,)\n",
            "log_probs looks like  (2219,)\n",
            "logs prob looks like  torch.Size([2219])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2219])\n",
            "rewards looks like  (1732,)\n",
            "log_probs looks like  (1732,)\n",
            "logs prob looks like  torch.Size([1732])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1732])\n",
            "rewards looks like  (1413,)\n",
            "log_probs looks like  (1413,)\n",
            "logs prob looks like  torch.Size([1413])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1413])\n",
            "rewards looks like  (1099,)\n",
            "log_probs looks like  (1099,)\n",
            "logs prob looks like  torch.Size([1099])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1099])\n",
            "rewards looks like  (1184,)\n",
            "log_probs looks like  (1184,)\n",
            "logs prob looks like  torch.Size([1184])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1184])\n",
            "rewards looks like  (1148,)\n",
            "log_probs looks like  (1148,)\n",
            "logs prob looks like  torch.Size([1148])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1148])\n",
            "rewards looks like  (1339,)\n",
            "log_probs looks like  (1339,)\n",
            "logs prob looks like  torch.Size([1339])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1339])\n",
            "rewards looks like  (2095,)\n",
            "log_probs looks like  (2095,)\n",
            "logs prob looks like  torch.Size([2095])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2095])\n",
            "rewards looks like  (1514,)\n",
            "log_probs looks like  (1514,)\n",
            "logs prob looks like  torch.Size([1514])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1514])\n",
            "rewards looks like  (1276,)\n",
            "log_probs looks like  (1276,)\n",
            "logs prob looks like  torch.Size([1276])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1276])\n",
            "rewards looks like  (1277,)\n",
            "log_probs looks like  (1277,)\n",
            "logs prob looks like  torch.Size([1277])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1277])\n",
            "rewards looks like  (1453,)\n",
            "log_probs looks like  (1453,)\n",
            "logs prob looks like  torch.Size([1453])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1453])\n",
            "rewards looks like  (1467,)\n",
            "log_probs looks like  (1467,)\n",
            "logs prob looks like  torch.Size([1467])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1467])\n",
            "rewards looks like  (1383,)\n",
            "log_probs looks like  (1383,)\n",
            "logs prob looks like  torch.Size([1383])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1383])\n",
            "rewards looks like  (1741,)\n",
            "log_probs looks like  (1741,)\n",
            "logs prob looks like  torch.Size([1741])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1741])\n",
            "rewards looks like  (1039,)\n",
            "log_probs looks like  (1039,)\n",
            "logs prob looks like  torch.Size([1039])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1039])\n",
            "rewards looks like  (1063,)\n",
            "log_probs looks like  (1063,)\n",
            "logs prob looks like  torch.Size([1063])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1063])\n",
            "rewards looks like  (1731,)\n",
            "log_probs looks like  (1731,)\n",
            "logs prob looks like  torch.Size([1731])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1731])\n",
            "rewards looks like  (2661,)\n",
            "log_probs looks like  (2661,)\n",
            "logs prob looks like  torch.Size([2661])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2661])\n",
            "rewards looks like  (704,)\n",
            "log_probs looks like  (704,)\n",
            "logs prob looks like  torch.Size([704])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([704])\n",
            "rewards looks like  (1389,)\n",
            "log_probs looks like  (1389,)\n",
            "logs prob looks like  torch.Size([1389])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1389])\n",
            "rewards looks like  (2131,)\n",
            "log_probs looks like  (2131,)\n",
            "logs prob looks like  torch.Size([2131])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2131])\n",
            "rewards looks like  (1779,)\n",
            "log_probs looks like  (1779,)\n",
            "logs prob looks like  torch.Size([1779])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1779])\n",
            "rewards looks like  (1415,)\n",
            "log_probs looks like  (1415,)\n",
            "logs prob looks like  torch.Size([1415])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1415])\n",
            "rewards looks like  (2320,)\n",
            "log_probs looks like  (2320,)\n",
            "logs prob looks like  torch.Size([2320])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2320])\n",
            "rewards looks like  (1147,)\n",
            "log_probs looks like  (1147,)\n",
            "logs prob looks like  torch.Size([1147])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1147])\n",
            "rewards looks like  (1022,)\n",
            "log_probs looks like  (1022,)\n",
            "logs prob looks like  torch.Size([1022])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1022])\n",
            "rewards looks like  (2141,)\n",
            "log_probs looks like  (2141,)\n",
            "logs prob looks like  torch.Size([2141])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2141])\n",
            "rewards looks like  (1362,)\n",
            "log_probs looks like  (1362,)\n",
            "logs prob looks like  torch.Size([1362])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1362])\n",
            "rewards looks like  (1450,)\n",
            "log_probs looks like  (1450,)\n",
            "logs prob looks like  torch.Size([1450])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1450])\n",
            "rewards looks like  (1546,)\n",
            "log_probs looks like  (1546,)\n",
            "logs prob looks like  torch.Size([1546])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1546])\n",
            "rewards looks like  (1166,)\n",
            "log_probs looks like  (1166,)\n",
            "logs prob looks like  torch.Size([1166])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1166])\n",
            "rewards looks like  (1647,)\n",
            "log_probs looks like  (1647,)\n",
            "logs prob looks like  torch.Size([1647])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1647])\n",
            "rewards looks like  (1205,)\n",
            "log_probs looks like  (1205,)\n",
            "logs prob looks like  torch.Size([1205])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1205])\n",
            "rewards looks like  (2098,)\n",
            "log_probs looks like  (2098,)\n",
            "logs prob looks like  torch.Size([2098])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2098])\n",
            "rewards looks like  (1940,)\n",
            "log_probs looks like  (1940,)\n",
            "logs prob looks like  torch.Size([1940])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1940])\n",
            "rewards looks like  (2191,)\n",
            "log_probs looks like  (2191,)\n",
            "logs prob looks like  torch.Size([2191])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2191])\n",
            "rewards looks like  (2740,)\n",
            "log_probs looks like  (2740,)\n",
            "logs prob looks like  torch.Size([2740])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2740])\n",
            "rewards looks like  (587,)\n",
            "log_probs looks like  (587,)\n",
            "logs prob looks like  torch.Size([587])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([587])\n",
            "rewards looks like  (1063,)\n",
            "log_probs looks like  (1063,)\n",
            "logs prob looks like  torch.Size([1063])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1063])\n",
            "rewards looks like  (861,)\n",
            "log_probs looks like  (861,)\n",
            "logs prob looks like  torch.Size([861])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([861])\n",
            "rewards looks like  (1051,)\n",
            "log_probs looks like  (1051,)\n",
            "logs prob looks like  torch.Size([1051])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1051])\n",
            "rewards looks like  (1389,)\n",
            "log_probs looks like  (1389,)\n",
            "logs prob looks like  torch.Size([1389])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1389])\n",
            "rewards looks like  (1152,)\n",
            "log_probs looks like  (1152,)\n",
            "logs prob looks like  torch.Size([1152])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1152])\n",
            "rewards looks like  (1103,)\n",
            "log_probs looks like  (1103,)\n",
            "logs prob looks like  torch.Size([1103])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1103])\n",
            "rewards looks like  (1887,)\n",
            "log_probs looks like  (1887,)\n",
            "logs prob looks like  torch.Size([1887])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1887])\n",
            "rewards looks like  (1753,)\n",
            "log_probs looks like  (1753,)\n",
            "logs prob looks like  torch.Size([1753])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1753])\n",
            "rewards looks like  (1372,)\n",
            "log_probs looks like  (1372,)\n",
            "logs prob looks like  torch.Size([1372])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1372])\n",
            "rewards looks like  (1056,)\n",
            "log_probs looks like  (1056,)\n",
            "logs prob looks like  torch.Size([1056])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1056])\n",
            "rewards looks like  (1465,)\n",
            "log_probs looks like  (1465,)\n",
            "logs prob looks like  torch.Size([1465])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1465])\n",
            "rewards looks like  (3297,)\n",
            "log_probs looks like  (3297,)\n",
            "logs prob looks like  torch.Size([3297])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([3297])\n",
            "rewards looks like  (2492,)\n",
            "log_probs looks like  (2492,)\n",
            "logs prob looks like  torch.Size([2492])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2492])\n",
            "rewards looks like  (1580,)\n",
            "log_probs looks like  (1580,)\n",
            "logs prob looks like  torch.Size([1580])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1580])\n",
            "rewards looks like  (1357,)\n",
            "log_probs looks like  (1357,)\n",
            "logs prob looks like  torch.Size([1357])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1357])\n",
            "rewards looks like  (1227,)\n",
            "log_probs looks like  (1227,)\n",
            "logs prob looks like  torch.Size([1227])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1227])\n",
            "rewards looks like  (2123,)\n",
            "log_probs looks like  (2123,)\n",
            "logs prob looks like  torch.Size([2123])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2123])\n",
            "rewards looks like  (1864,)\n",
            "log_probs looks like  (1864,)\n",
            "logs prob looks like  torch.Size([1864])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1864])\n",
            "rewards looks like  (1324,)\n",
            "log_probs looks like  (1324,)\n",
            "logs prob looks like  torch.Size([1324])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1324])\n",
            "rewards looks like  (1281,)\n",
            "log_probs looks like  (1281,)\n",
            "logs prob looks like  torch.Size([1281])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1281])\n",
            "rewards looks like  (1366,)\n",
            "log_probs looks like  (1366,)\n",
            "logs prob looks like  torch.Size([1366])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1366])\n",
            "rewards looks like  (957,)\n",
            "log_probs looks like  (957,)\n",
            "logs prob looks like  torch.Size([957])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([957])\n",
            "rewards looks like  (1187,)\n",
            "log_probs looks like  (1187,)\n",
            "logs prob looks like  torch.Size([1187])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1187])\n",
            "rewards looks like  (1625,)\n",
            "log_probs looks like  (1625,)\n",
            "logs prob looks like  torch.Size([1625])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1625])\n",
            "rewards looks like  (1605,)\n",
            "log_probs looks like  (1605,)\n",
            "logs prob looks like  torch.Size([1605])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1605])\n",
            "rewards looks like  (1015,)\n",
            "log_probs looks like  (1015,)\n",
            "logs prob looks like  torch.Size([1015])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1015])\n",
            "rewards looks like  (1565,)\n",
            "log_probs looks like  (1565,)\n",
            "logs prob looks like  torch.Size([1565])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1565])\n",
            "rewards looks like  (1353,)\n",
            "log_probs looks like  (1353,)\n",
            "logs prob looks like  torch.Size([1353])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1353])\n",
            "rewards looks like  (1321,)\n",
            "log_probs looks like  (1321,)\n",
            "logs prob looks like  torch.Size([1321])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1321])\n",
            "rewards looks like  (1074,)\n",
            "log_probs looks like  (1074,)\n",
            "logs prob looks like  torch.Size([1074])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1074])\n",
            "rewards looks like  (1301,)\n",
            "log_probs looks like  (1301,)\n",
            "logs prob looks like  torch.Size([1301])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1301])\n",
            "rewards looks like  (2105,)\n",
            "log_probs looks like  (2105,)\n",
            "logs prob looks like  torch.Size([2105])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2105])\n",
            "rewards looks like  (2008,)\n",
            "log_probs looks like  (2008,)\n",
            "logs prob looks like  torch.Size([2008])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2008])\n",
            "rewards looks like  (1885,)\n",
            "log_probs looks like  (1885,)\n",
            "logs prob looks like  torch.Size([1885])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1885])\n",
            "rewards looks like  (1184,)\n",
            "log_probs looks like  (1184,)\n",
            "logs prob looks like  torch.Size([1184])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1184])\n",
            "rewards looks like  (2551,)\n",
            "log_probs looks like  (2551,)\n",
            "logs prob looks like  torch.Size([2551])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2551])\n",
            "rewards looks like  (1330,)\n",
            "log_probs looks like  (1330,)\n",
            "logs prob looks like  torch.Size([1330])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1330])\n",
            "rewards looks like  (1510,)\n",
            "log_probs looks like  (1510,)\n",
            "logs prob looks like  torch.Size([1510])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1510])\n",
            "rewards looks like  (1330,)\n",
            "log_probs looks like  (1330,)\n",
            "logs prob looks like  torch.Size([1330])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1330])\n",
            "rewards looks like  (2157,)\n",
            "log_probs looks like  (2157,)\n",
            "logs prob looks like  torch.Size([2157])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2157])\n",
            "rewards looks like  (1276,)\n",
            "log_probs looks like  (1276,)\n",
            "logs prob looks like  torch.Size([1276])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1276])\n",
            "rewards looks like  (1188,)\n",
            "log_probs looks like  (1188,)\n",
            "logs prob looks like  torch.Size([1188])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1188])\n",
            "rewards looks like  (2381,)\n",
            "log_probs looks like  (2381,)\n",
            "logs prob looks like  torch.Size([2381])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2381])\n",
            "rewards looks like  (1450,)\n",
            "log_probs looks like  (1450,)\n",
            "logs prob looks like  torch.Size([1450])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1450])\n",
            "rewards looks like  (1612,)\n",
            "log_probs looks like  (1612,)\n",
            "logs prob looks like  torch.Size([1612])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1612])\n",
            "rewards looks like  (1780,)\n",
            "log_probs looks like  (1780,)\n",
            "logs prob looks like  torch.Size([1780])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1780])\n",
            "rewards looks like  (1350,)\n",
            "log_probs looks like  (1350,)\n",
            "logs prob looks like  torch.Size([1350])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1350])\n",
            "rewards looks like  (1459,)\n",
            "log_probs looks like  (1459,)\n",
            "logs prob looks like  torch.Size([1459])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1459])\n",
            "rewards looks like  (1958,)\n",
            "log_probs looks like  (1958,)\n",
            "logs prob looks like  torch.Size([1958])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1958])\n",
            "rewards looks like  (1325,)\n",
            "log_probs looks like  (1325,)\n",
            "logs prob looks like  torch.Size([1325])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1325])\n",
            "rewards looks like  (2168,)\n",
            "log_probs looks like  (2168,)\n",
            "logs prob looks like  torch.Size([2168])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2168])\n",
            "rewards looks like  (1682,)\n",
            "log_probs looks like  (1682,)\n",
            "logs prob looks like  torch.Size([1682])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1682])\n",
            "rewards looks like  (852,)\n",
            "log_probs looks like  (852,)\n",
            "logs prob looks like  torch.Size([852])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([852])\n",
            "rewards looks like  (1757,)\n",
            "log_probs looks like  (1757,)\n",
            "logs prob looks like  torch.Size([1757])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1757])\n",
            "rewards looks like  (2313,)\n",
            "log_probs looks like  (2313,)\n",
            "logs prob looks like  torch.Size([2313])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2313])\n",
            "rewards looks like  (1662,)\n",
            "log_probs looks like  (1662,)\n",
            "logs prob looks like  torch.Size([1662])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1662])\n",
            "rewards looks like  (1559,)\n",
            "log_probs looks like  (1559,)\n",
            "logs prob looks like  torch.Size([1559])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1559])\n",
            "rewards looks like  (2077,)\n",
            "log_probs looks like  (2077,)\n",
            "logs prob looks like  torch.Size([2077])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2077])\n",
            "rewards looks like  (2119,)\n",
            "log_probs looks like  (2119,)\n",
            "logs prob looks like  torch.Size([2119])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2119])\n",
            "rewards looks like  (954,)\n",
            "log_probs looks like  (954,)\n",
            "logs prob looks like  torch.Size([954])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([954])\n",
            "rewards looks like  (1797,)\n",
            "log_probs looks like  (1797,)\n",
            "logs prob looks like  torch.Size([1797])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1797])\n",
            "rewards looks like  (1579,)\n",
            "log_probs looks like  (1579,)\n",
            "logs prob looks like  torch.Size([1579])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1579])\n",
            "rewards looks like  (1277,)\n",
            "log_probs looks like  (1277,)\n",
            "logs prob looks like  torch.Size([1277])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1277])\n",
            "rewards looks like  (1196,)\n",
            "log_probs looks like  (1196,)\n",
            "logs prob looks like  torch.Size([1196])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1196])\n",
            "rewards looks like  (1294,)\n",
            "log_probs looks like  (1294,)\n",
            "logs prob looks like  torch.Size([1294])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1294])\n",
            "rewards looks like  (1318,)\n",
            "log_probs looks like  (1318,)\n",
            "logs prob looks like  torch.Size([1318])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1318])\n",
            "rewards looks like  (2605,)\n",
            "log_probs looks like  (2605,)\n",
            "logs prob looks like  torch.Size([2605])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2605])\n",
            "rewards looks like  (2002,)\n",
            "log_probs looks like  (2002,)\n",
            "logs prob looks like  torch.Size([2002])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2002])\n",
            "rewards looks like  (1354,)\n",
            "log_probs looks like  (1354,)\n",
            "logs prob looks like  torch.Size([1354])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1354])\n",
            "rewards looks like  (1785,)\n",
            "log_probs looks like  (1785,)\n",
            "logs prob looks like  torch.Size([1785])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1785])\n",
            "rewards looks like  (781,)\n",
            "log_probs looks like  (781,)\n",
            "logs prob looks like  torch.Size([781])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([781])\n",
            "rewards looks like  (1965,)\n",
            "log_probs looks like  (1965,)\n",
            "logs prob looks like  torch.Size([1965])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1965])\n",
            "rewards looks like  (1135,)\n",
            "log_probs looks like  (1135,)\n",
            "logs prob looks like  torch.Size([1135])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1135])\n",
            "rewards looks like  (1672,)\n",
            "log_probs looks like  (1672,)\n",
            "logs prob looks like  torch.Size([1672])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1672])\n",
            "rewards looks like  (1278,)\n",
            "log_probs looks like  (1278,)\n",
            "logs prob looks like  torch.Size([1278])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1278])\n",
            "rewards looks like  (2499,)\n",
            "log_probs looks like  (2499,)\n",
            "logs prob looks like  torch.Size([2499])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2499])\n",
            "rewards looks like  (1275,)\n",
            "log_probs looks like  (1275,)\n",
            "logs prob looks like  torch.Size([1275])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1275])\n",
            "rewards looks like  (1144,)\n",
            "log_probs looks like  (1144,)\n",
            "logs prob looks like  torch.Size([1144])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1144])\n",
            "rewards looks like  (1605,)\n",
            "log_probs looks like  (1605,)\n",
            "logs prob looks like  torch.Size([1605])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1605])\n",
            "rewards looks like  (1178,)\n",
            "log_probs looks like  (1178,)\n",
            "logs prob looks like  torch.Size([1178])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1178])\n",
            "rewards looks like  (3269,)\n",
            "log_probs looks like  (3269,)\n",
            "logs prob looks like  torch.Size([3269])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([3269])\n",
            "rewards looks like  (1492,)\n",
            "log_probs looks like  (1492,)\n",
            "logs prob looks like  torch.Size([1492])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1492])\n",
            "rewards looks like  (1285,)\n",
            "log_probs looks like  (1285,)\n",
            "logs prob looks like  torch.Size([1285])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1285])\n",
            "rewards looks like  (1687,)\n",
            "log_probs looks like  (1687,)\n",
            "logs prob looks like  torch.Size([1687])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1687])\n",
            "rewards looks like  (1124,)\n",
            "log_probs looks like  (1124,)\n",
            "logs prob looks like  torch.Size([1124])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1124])\n",
            "rewards looks like  (2043,)\n",
            "log_probs looks like  (2043,)\n",
            "logs prob looks like  torch.Size([2043])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2043])\n",
            "rewards looks like  (1280,)\n",
            "log_probs looks like  (1280,)\n",
            "logs prob looks like  torch.Size([1280])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1280])\n",
            "rewards looks like  (1418,)\n",
            "log_probs looks like  (1418,)\n",
            "logs prob looks like  torch.Size([1418])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1418])\n",
            "rewards looks like  (1365,)\n",
            "log_probs looks like  (1365,)\n",
            "logs prob looks like  torch.Size([1365])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1365])\n",
            "rewards looks like  (1091,)\n",
            "log_probs looks like  (1091,)\n",
            "logs prob looks like  torch.Size([1091])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1091])\n",
            "rewards looks like  (1279,)\n",
            "log_probs looks like  (1279,)\n",
            "logs prob looks like  torch.Size([1279])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1279])\n",
            "rewards looks like  (1109,)\n",
            "log_probs looks like  (1109,)\n",
            "logs prob looks like  torch.Size([1109])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1109])\n",
            "rewards looks like  (1285,)\n",
            "log_probs looks like  (1285,)\n",
            "logs prob looks like  torch.Size([1285])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1285])\n",
            "rewards looks like  (1222,)\n",
            "log_probs looks like  (1222,)\n",
            "logs prob looks like  torch.Size([1222])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1222])\n",
            "rewards looks like  (1538,)\n",
            "log_probs looks like  (1538,)\n",
            "logs prob looks like  torch.Size([1538])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1538])\n",
            "rewards looks like  (1139,)\n",
            "log_probs looks like  (1139,)\n",
            "logs prob looks like  torch.Size([1139])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1139])\n",
            "rewards looks like  (1354,)\n",
            "log_probs looks like  (1354,)\n",
            "logs prob looks like  torch.Size([1354])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1354])\n",
            "rewards looks like  (1166,)\n",
            "log_probs looks like  (1166,)\n",
            "logs prob looks like  torch.Size([1166])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1166])\n",
            "rewards looks like  (1348,)\n",
            "log_probs looks like  (1348,)\n",
            "logs prob looks like  torch.Size([1348])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1348])\n",
            "rewards looks like  (1347,)\n",
            "log_probs looks like  (1347,)\n",
            "logs prob looks like  torch.Size([1347])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1347])\n",
            "rewards looks like  (2059,)\n",
            "log_probs looks like  (2059,)\n",
            "logs prob looks like  torch.Size([2059])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2059])\n",
            "rewards looks like  (2021,)\n",
            "log_probs looks like  (2021,)\n",
            "logs prob looks like  torch.Size([2021])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2021])\n",
            "rewards looks like  (2232,)\n",
            "log_probs looks like  (2232,)\n",
            "logs prob looks like  torch.Size([2232])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2232])\n",
            "rewards looks like  (1102,)\n",
            "log_probs looks like  (1102,)\n",
            "logs prob looks like  torch.Size([1102])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1102])\n",
            "rewards looks like  (1165,)\n",
            "log_probs looks like  (1165,)\n",
            "logs prob looks like  torch.Size([1165])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1165])\n",
            "rewards looks like  (1264,)\n",
            "log_probs looks like  (1264,)\n",
            "logs prob looks like  torch.Size([1264])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1264])\n",
            "rewards looks like  (1346,)\n",
            "log_probs looks like  (1346,)\n",
            "logs prob looks like  torch.Size([1346])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1346])\n",
            "rewards looks like  (2848,)\n",
            "log_probs looks like  (2848,)\n",
            "logs prob looks like  torch.Size([2848])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2848])\n",
            "rewards looks like  (938,)\n",
            "log_probs looks like  (938,)\n",
            "logs prob looks like  torch.Size([938])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([938])\n",
            "rewards looks like  (1069,)\n",
            "log_probs looks like  (1069,)\n",
            "logs prob looks like  torch.Size([1069])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1069])\n",
            "rewards looks like  (2588,)\n",
            "log_probs looks like  (2588,)\n",
            "logs prob looks like  torch.Size([2588])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2588])\n",
            "rewards looks like  (1461,)\n",
            "log_probs looks like  (1461,)\n",
            "logs prob looks like  torch.Size([1461])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1461])\n",
            "rewards looks like  (2153,)\n",
            "log_probs looks like  (2153,)\n",
            "logs prob looks like  torch.Size([2153])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2153])\n",
            "rewards looks like  (2312,)\n",
            "log_probs looks like  (2312,)\n",
            "logs prob looks like  torch.Size([2312])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2312])\n",
            "rewards looks like  (1636,)\n",
            "log_probs looks like  (1636,)\n",
            "logs prob looks like  torch.Size([1636])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1636])\n",
            "rewards looks like  (2019,)\n",
            "log_probs looks like  (2019,)\n",
            "logs prob looks like  torch.Size([2019])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2019])\n",
            "rewards looks like  (1450,)\n",
            "log_probs looks like  (1450,)\n",
            "logs prob looks like  torch.Size([1450])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([1450])\n",
            "rewards looks like  (2105,)\n",
            "log_probs looks like  (2105,)\n",
            "logs prob looks like  torch.Size([2105])\n",
            "torch.from_numpy(rewards) looks like  torch.Size([2105])\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vNb_tuFYhKVK"
      },
      "source": [
        "### Training Result\n",
        "During the training process, we recorded `avg_total_reward`, which represents the average total reward of episodes before updating the policy network.\n",
        "\n",
        "Theoretically, if the agent becomes better, the `avg_total_reward` will increase.\n",
        "The visualization of the training process is shown below:  \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        },
        "id": "wZYOI8H10SHN",
        "outputId": "54840043-6fe4-4771-e8c9-78785c55aa79"
      },
      "source": [
        "plt.plot(avg_total_rewards)\n",
        "plt.title(\"Total Rewards\")\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAEICAYAAAC3Y/QeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOx9d9wdRb3+M7unvD1vOglJSCChhA6hKF2Rjnj1ykW8XGwXC3ivehXhooIgyrUXwJ+oKCJSFEEUkCZNlJJQQoAQQnqvb9562u78/tid2dnZmd0973vOW+f5fJL3nN09s7N7zj7znedbhlBKYWBgYGAwtmANdQcMDAwMDAYfhvwNDAwMxiAM+RsYGBiMQRjyNzAwMBiDMORvYGBgMAZhyN/AwMBgDMKQv8GYByGEEkLmDnU/+gtCyImEkHVD3Q+DkQVD/gbDFoSQbuGfSwjpE95/WPOZmhIhIeQJQkjBP+c2QsgfCSHTatW+gcFQwZC/wbAFpbSF/QOwBsDZwrbbBrErl/h9mAugBcB3B/HcIRBCMkN1boPRBUP+BiMOhJA8IeSHhJAN/r8f+tuaATwIYLowQ5hOCDmSEPJPQkgHIWQjIeR6Qkiu2vNSSjsA3AvgEKEv+xJCHiGE7CCEvEkIOdffPsc/n+W//zkhZIvwuVsJIZ/zX3+UEPIGIaSLELKCEPJJ4bgTCSHrCCFfJoRsAvArQkgjIeTXhJCdhJDXARwh3Z8vE0LW++29SQh5d7XXajD6YcjfYCTiCgBHwyPhgwEcCeArlNIeAKcD2CDMEDYAcAB8HsAkAO8A8G4An6n2pISQiQDeD2C5/74ZwCMAfgdgCoDzANxICJlPKV0JoBPAof7HjwfQTQjZz39/AoAn/ddbAJwFoA3ARwH8gBBymHDq3QBMALAHgIsAXAlgL//fqQAuFPq4D4BLABxBKW3196+q9loNRj8M+RuMRHwYwNWU0i2U0q0Avg7gAt3BlNJFlNJnKaUVSukqAD+DR75p8WNCyC4A2+ANIJ/1t58FYBWl9Fd+2y8BuBvAB/39TwI4gRCym//+D/77OfCI/hW/f/dTSt+mHp4E8DCA44TzuwCupJQWKaV9AM4FcC2ldAeldC2AHwvHOgDyAOYTQrKU0lWU0reruFaDMQJD/gYjEdMBrBber/a3KUEI2ZsQ8hdCyCZCSCeAb8Ij8bT4L0rpOAAHARgPYIa/fQ8AR/nyTgchpAPewMTI/kkAJ8Kz+p8C8AS8QecEAE9TSl2/f6cTQp71paMOAGdI/dtKKS1I179Wun4AAKV0OYDPAbgKwBZCyB2EEO29MRi7MORvMBKxAR7xMszytwGAqkztTwEsBTCPUtoG4H8BkGpPSil9FcA3ANxACCHwCPhJSmm78K+FUvpp/yNPwrPgT/Rf/x3AMRAkH0JIHt5s4bsAplJK2wE8IPVPvqaNAGYK72dJ/fwdpfRYePeIAvi/aq/VYPTDkL/BSMTtAL5CCJlMCJkE4GsAfuvv2wxgIiFknHB8Kzz9vZsQsi+AT6P/uAXAVADvBfAXAHsTQi4ghGT9f0cwXZ9S+haAPgD/Dm+Q6PT79wEEen8OnkyzFUCFEHI6gFMS+nAXgMsJIeMJITMQyFAghOxDCHmXP6gU/PO7A7heg1EKQ/4GIxHfALAQwGIArwJ40d8GSulSeIPDCl+KmQ7giwDOB9AF4OcA7uzviSmlJQA/AvBVSmkXPKI+D97MYxM8KzsvfORJANt9bZ69J36f4bfxX/AIfaffz/sSuvF1eFLPSnj+gVuFfXkA18HzT2yC54i+vB+XajDKQcxiLgYGBgZjD8byNzAwMBiDMORvYGBgMAZhyN/AwMBgDMKQv4GBgcEYxIgoEjVp0iQ6e/bsoe6GgYGBwYjCokWLtlFKJ6v2jQjynz17NhYuXDjU3TAwMDAYUSCErNbtM7KPgYGBwRiEIX8DAwODMQhD/gYGBgZjEIb8DQwMDMYgDPkbGBgYjEEY8jcwMDAYgzDkb2BgYDAGYcjfwMDAoJ+496X16ClWhrob/YIhfwMDA4N+4KU1O/G5O1/GV+9dMtRd6RcM+RsYDGM4LsVX7n0Va7b31qX9HT0lPPbG5rq0PdrRU3QAAJu7CglHDk8Y8jcwGMZ4ZV0HfvvsGvz3nS/Vpf2P3/ICPn7LQnQWynVpfySgo7eEtTuqH1ypcrnokQND/gYGIwBunXhmxdYer/16nWAE4KTvPoHjvv141Z9jiyASkBr3aHBgyN/AYBiD00qdlls1y7gCO3v7N+thd46MTO435G9gMJxBfGapF0Ub6h+7MORvYDCMwYzKuhnofrvOGJZ9xioM+RsYDGMwSaFezkXWquH+6jHSJTND/gYGwxjMmVgvnmEE5o5wIhtKkBEq+hvyNzAYxuCWf5252cg+1WOk3zFD/gYGwxiB7FMfBLLPSKeyIQAP9RyZMORvYDCMEcg+9Qr19P66bl2aH3X408vrsXxLF4DADzNCVZ+RsYC7gcFYRb2JhRGYsfzT4b/veBkAsOq6M/m2Ecr9xvI3MBgJqJ/D1/vrGPKvGrW+ZY+9sRm7+plw1h8Y8jcwGMZgFnndQz2Nw7dq8PIONZiebekq4OO3LMTFv3txwG2lRU3InxByMyFkCyFkibBtAiHkEULIW/7f8f52Qgj5MSFkOSFkMSHksFr0wcBgNIIRTL0Nc8P9yZD9Lry8Qw3a7it5FUJX7+ipQWvpUCvL/9cATpO2XQbgMUrpPACP+e8B4HQA8/x/FwH4aY36YGAw6sDJv24n8P6YUM/k2Y98j9hgUAu/DGvbHkTvcU3In1L6FIAd0uZzANziv74FwPuE7b+hHp4F0E4ImVaLfhgYjDYwuadu0T7G4cuRdA8qEvnXcrxkbVnWCCN/DaZSSjf6rzcBmOq/3h3AWuG4df62EAghFxFCFhJCFm7durWO3TQwGL5w62z581BPQ/6JTu8o+dfunrG2RpzlnwTqmS1V3SlK6U2U0gWU0gWTJ0+uU88MDIY3mMVft2gf/+9oVH3uX7wRF/1mYerjg5wHqpSAHCe8LZCBBk7YFb9texAt/3rG+W8mhEyjlG70ZZ0t/vb1AGYKx83wtxkYGEjgln+dLfPRqPlXGznD7sG8rzyIeVNa8NfPHR/aX5Ey4dwaav5BW6PD8r8PwIX+6wsB/EnY/h9+1M/RAHYJ8pCBgUEIVPi/Dq2bwm4c7B44LsXSTV2R/YMh+2Qky3/jrj5s6Oir2XlE1MTyJ4TcDuBEAJMIIesAXAngOgB3EUI+DmA1gHP9wx8AcAaA5QB6AXy0Fn0wMBiNCCz/+rRv4vwDJJW4kMnf8Y+vha3OZh2yw/fLd7+Kzr4y7r34mBqcJYyakD+l9EOaXe9WHEsBXFyL8xoYjHYEoZ71re1jMnyTLXlZ82cDZi1lH1tqi1KKerkBTIavgcEwRhwhrdrWg3lXPIC3t3YP+DyjmfvT+kuSo300mn8dHb6OS2HVyQ9gyN/AYBjDjYn2+fMrG1B2KP744roBn2c0OnwZ0g5s1cb513K2VOGziDDRu5TWLfbfkL+BwXBGjObPSKEWHDRSZZ/7F2/E7MvuR2dBXxAt7bW5bvwsoVJH2aeiyfB1KYzsY2AwFhFnkDOeqIXRPpLWo3Vcysn+xieWAwBWb+vVHp82KselFMWK3usrz46cWpK/7z2WZR/XyD4GBmMTceUX+rPQS8Vxcf/ijZHoHmcELeZy7f1v4KCrHkah7HBijHOIp709jhtP/uWI5p+u3TQoO+poH5fSuiV+GfI3MKgDlm3uwpf/sHjAWnpcqCfjhGrO8McX1+Pi372I3z63WjrPyLH8b39+DQCg5Lj8HsTd5rTfAaVAseKkbqeW9ywo7Bbe7tL6JX4Z8jcw6AcKZQelGCvxP3+zEHcuXIu1O/RyRBrw8g4KimdWbzUx+syKfH5luA7jSIrz7yt7BE3dgBjjiDgtSTuUoljWf6ey5s9ln1pE+7hq2ceEehoYDDPs+9W/4j0/eFK7v6foEVRjzh7QeeLq+fdH8x/fnAUALN8SDg8diQ5fh9JggftY8k/XXrWafw1L+/CBRdb3HVo/zd+s4Wtg0E+s3q636ntLFQAD5wUaU94hjdUrg2n7MvmPIMOfQ4yBj+t/Wp+IW7XmX8tQT53DNzog1AqG/A3GFJ5dsR2HzGxHQ3ZgFnkSev2VmQZCD5+9/SX8+ZUNXjsqy9//W43DlxFWxaX408tBPcWRJPswuJTy0Mi4/qfV/F2KeMtfK/sMHCzUU+XwNbKPgcEAsXxLF8676Vl8/c+vDdo5B2IdMuL3EG2HtV0Nb4sDxW3Prom0JeOHjy7D7xeuVe4balRcypk37h6kvT9etI/e4Rut7ZNciXNXbxkdvaXEc+tkH7eOso8hf4Mxg45eLzb8TUXFxnqhVga1yM2/eHoFjrz2UR4eWM0Aw/ozpTUfkjF01vF9r2zAY29sUe4bangx8N7ruNlPatknQfOXyzsEay3o2z/46odxyNWPxJ73H29vw/WPe/kKqmifeoV6GtnHwKBKVCOz1Cp5SmzlG/e/ASBIDKpmgGEDRda2QoSvG0Bclw5bZ7Co+ctWeei4asg/JtpHvF+UBvdloLfn/J8/x1+rZJ96lfg3lr/BmAGPDElx7IaOPvzy7yuV+8pONeSf+tCEdqINlRn5V8H+jMDyWSt0HbomKi4dVnV/xPsgRsLE9TF9tE98nL8Y6ll2KHee19LxGynvYDJ8DQxqgfQP0cd+/QKu+cvr2NJZiOwrxBCEjFoRA2vlZmFAKvusVqoiPZd1J2dbfOYA6MnTFcj/intexbcefKOKXtcezJEOeH1jvFiOuQdpB0fHTW/5Oy4VfC41JP+I5V8/2ceQv4GBAuv91ZNUckKhlJ78a2f5A5t2FXD1X17n28q+Ph2XbCaDEVU+Y4WuTSdPiZb/bc+twc+eXFF132uJrkKFv05v+afN8I13+Io+krLr8kGllhMj2cg3so+BQQ2Rhgv6fIIvlKNkUIixDmXUzPKnNNIWm4Fs6y7itB8+laq0MyOqXMYKWctay58OL9mnS6je6QgO33INZB+xto/K2hbvQ8UJNP9ahsnKq4lRWr84f0P+BmMGSc/Qls4CHn5tE4DA4lcRfZLsE3akVtlJDSiiJMAkiudW7sDSTV144NVNie2wASSXCTt8dW6MajX/bd1FrNzWk/r4atFVFCx/l/IwSydmDcb0VT0RS/6i5l8JWf61I3+5jIc4wNUahvwNDHxc+KsXcNGti0LWvoroVbMBET2livCuduwvE4Mclji1LZ/YDCOsnB12+OpkH0cR7fPF378SssBF/OCRZfjELS8AAG54fDne9d0nEvtUDUTJLWT5xzjhqwr19L9b2fEKhEM9Q5Z/DSdGclumqqeBQQ2he1Y3+87dbsG6VBF9X4Lm3y3o0rW0/OXCYrLWH+f0lPuTy1ghfVtn3Tsujfg9/rBoHX7zz9XK47sKFez08ym+89CbWFHjWYA4GHt6uB/qGUP+8m35x/JteN8Nz0TulxjnrwoPrUiyD3v74pqduKtGiXDyLMJU9TQwqAGSDMDmvFfyQSRvpeaf4GDtKYrkH53GP/3W1qSuRqDS3uVZSRrHbyD72KGZg26Qclyq1LR1fOS4NDI4phmUVNjeXcSS9btC28RoHMcNylpXI/u8sm4XXl7bEXIee20E5K+6Zkcj+3QVKrj0D4tTXFEKKCx/I/sYGKTA029txZPL1OSapM0257ycxx1COr5S80+QfcTQS/mU1/9tOS745fP4+1vbYtuQQWnUGpXDEtPkHzAJJGdbocFCd29Ulj+glkUAjxT7yk6IPLslkk0D16V4343P4Kyf/D20vVCRZR+vH3HXLl8bKwktD0qUBjOgRMtf4QtJ+l2kQdTyp9p7PVAY8jcYVbjgl8/jwpufV+7jhKQhupa8R/5iDX51tE/8Qy42Lz/MyzZ7pSV2pKj3EmoTUbKRwxLjShME/fH+5jLhR18r+1C15a/ToVk7Yl9EGS0NdvWWsef/PoC1O7xwW1GzFwdjse5NbKindFvY9yfPlMTYfUqjvgJR8y87bmSA2NZdjL2uNHCpl2C4aVfB7zsdfbIPIeQ0QsibhJDlhJDLhqofBmMHSWn+zT75r9kukn+85a9yJoqEL+9ms4KcXf2jJ+vaMtmnkVd4hq9E/irL33UpKI3WtAH0OjSzjnsFp3fc4uoqbOkKJ9aJ11kshy1/nuRVhezDZKmS44a+P1cKp5UHFNHyd/x7I2JrVzz5P/r65kTfAAXwzuv+hqO/9Zj3frSFehJCbAA3ADgdwHwAHyKEzB+KvhiMHcTwA4DA8hctuKQ4f5XFGa4BE97HrE2ZfJMgShIM/SF/MdQztF11HTHRLHIBMv4Z/+C+soOGrHeOamUfmevErF7R3xKq7dNP2UeubyT+RmRjwYmUd5Atf/1sjlKK3z2/Br98eqX2GFVfnVGo+R8JYDmldAWltATgDgDnDFFfDMYI+JRes585fDtFh29CqKeshz+3YjseXxpUwfz9orV4ac1O/p6Rv0y+SaBQaP5C33IZK5XDVyzvIEJF8IzcKq4bITq5ABkDI+FC2eFrJlQr+8izCnEWUdBY/vL3ELbow+1z8q/Q0D6XhslXNhbC0T5R2SfO8q+4FGXHTRyg5ZlkPUM9h6qq5+4AxPnPOgBHiQcQQi4CcBEAzJo1a/B6ZjBqkST7NPkO386+QKYQ48p7ihVkbMLJA/CsR3FhmH+76dlQm7/552oUyy4OnTUeQCD7VP0402hEi+jwndKaH5Dlr7o3jPBdNzqr0PXf4bKPw2c3clRNtRCjh0KzLiECquJE9XuGSGY0l32ciMwj3gf5nlSkEtgyUceSv0NRqriJfhn5axiToZ6U0psopQsopQsmT5481N0xGAVISsPP+lpGV8jyDx7W/a98CO/9yTMhAoqTG/h5hSe6FBNHHgcKGqv5T2nND8jhq9L8K4LlL5O/zsHKCLKvFFj+XVVa/nLbvSUHb/mOcnG24woO2jh9Xv7emYxUqtCIc158r1u5C/DKSURlHz35l10XpRSWfyTaZxRm+K4HMFN4P8PfZmBQNyQE+/D9u0TLX9L839zcFYrjj3M0yu0CAfnH1Z9XQa35B32b1FKl5S/LPor+sG2OGw2l1IVWcsu/7KAhE82bSAN5kLvjhTV4zw+ewt+Wbo5Y/uz2y/2JK7Ehav4hmUd2+MqWvxjn77iR5DEWwXX3onX465JNoXvqOJ7sI1dg1S4KL/RptGX4vgBgHiFkDiEkB+A8APcNUV8MxgiSatSwB1+MTlE5fJdu6uSvP3/ny3ww0JGvKA+wY2SrMgkujZZZKFZcnH7Abnjj6tPQnM+kKu3MyztEQj2jx1Y4+bsRWUU36PGaSCWH+wV0pSB0kL+nhas8n8mbm7oj0T4ODfqo6gegkH1Eh29I9gmTb9xsouxEC+2xKqv/8/tX8KnfLgof77ooVyg/JmgnmmsgYtTJPpTSCoBLADwE4A0Ad1FKB29hVYMxicDhqyZe9uCFNH9FqOeS9Z1oa/D8A88s346b/74SfSUH2zXRHiJJxJUPiAOFyqnpkXhjzkbWJihX0khQ3l+V7PPGxk585rZFwSIxgqQiDyy6c4maPxswqnX4ygMLc7pTeBm4TJ5zhRwEuapnnOYvWv5UOJXYHgD8+LG3sKuvzB3O8oAit6tb4xfwNX+F5S9LdSFHtf/50Sb7gFL6AKV0b0rpXpTSa4eqHwajE0oZQ0jgUYE9eD0hB6Oitk/ZweF7jOfvv/fIMiz4xiNYuHqHui+i7MMs/4RZiOxMpFQ9W8hY3iOcy1jpLH+/XVWc/xfuegUPvLqJr3EcWP5Rf4Mq9l/8TF/Z4a/Tyj6vrO3w5RSJrP3vg1Lv+2CO+YpgfVf8mP2dPaVI/yKyj99esSLJPm6Y0G99djUO/nqwBq/jujy6yFFo/rIVLzuISxUXZYcqZ4IMYousLybD18CgCqiIMJFwFdv6NNm886e3hd73lBxc8ruXlMe6ioc9ri/v+NZj+PZDb0a2q2YLzArO2lZEUlCBUi88kg0aYh9zNiuVEK5v41AaISndQMPj/EsO92+kcfguWb8L59zwDH7w6LLIQBOK9qk4aMp5voQ/vrgeb2wMBqq/L9+GI7/5KLZ2FVNa/mGyl0M9+bUyP41Dua/EcRWyj0z+Tvh7Z/vFeyeH58r9AfRhtQOFIX+DUQmV/p5U20feb1sExbKLjt4SfvtsuIolsz7TQGyWR/vEkP/GXQX89Im3I9tVn8nageVflK75zU1d+NvSzeE2/JIIshPRcYO2ShUXL6zageO+/Tg/r+xQ1UU58WifssNfs35TSjH7svvxnYeWRj63bqeXVb1sc3fU8i87/PPFssvJ/58rtmOTX4m14lBs7iyi7FDs6iuF+ifPArWaP6VK30dwbZTLZSrLPzo7oqHXjPTLTtQYCKSs4PPs92hW8jIwqAKqhCc2CxfJuFhxsMOXCmRubczaKFQcfPnuxfjKvUtC+6rJ0FWGelYZ7QOoI4QyPml49fnD5QpO/eFT+NivF0p98TTkrJSi6wrEVqy4+NGjb/F9jhu1/HXObSZN9ZYcTnJBspj394bHowMbI/imnB3R/MXLLlQcXoYjdF6XcqevXHRN/LxngVP+WvwtUBqN3RdRcSnyfgSTakCMC4ctO65y2U32+vvnHoJDZ7VHyk0Ao6y8g4FBvaGUfRQP9md++yIOu8bTdOXdrQ0Z7Oor88FBRDUZuiL5izp6tVCVLeaWv20pw0FVfbEIQSaS4Ut5W72lSkR+SEv+wQpoTkTiisuJYLH3jVlbGwnlUs8B36yYdZUdN8hLcKg22kf04ZQkzV8l5YTO7wbS2KV3L8ai1TtD+yOzI0nzFwcdhqKQ8W0RIuUdeH+N5m9gIMFxKRZpnKyqaBSVE/gxvxSDq8jYnDulBWt39CqTp3SF2eZNaYmeV8En1Ub7AOpwTK75+4NRktOXFQrLyrIPDSz/nqKjKFecrnx0EO1T4STHSDAuJ4Lp+o05W5sDUaq4vsPXjuwTrX1ZkhG/d9GHU3LcSHmHuAhcR5gdqSA7wWXLn303ouXP7lHOtkAga/5G9jEwUOJnT72ND/z0n/jn29sj+0pO1FHLk7wUbfWVnYjlv9+0NrgUWLE1uhqVjgSyikFBJSVUm+QFqC1/Hu3jnzcp3JNljMqWP6VBGz2lSuReyFFPSZZ/X9nlln4ay79PsPx1kUSFsoNixUWTQvapCBFJFdfVRvsUSgLxVsKhnSoDgIHlFKi+X95ejObfJ5xX5fBllr9q0DKyj4GBhLc2dwPw6p/LUFnrcdZ2b8mJTPn33a0VgDpOXU/+0QeVPc/iCl66UhNJmrPufMzyLyoGPbkvnuwT9DNre6TD2uouViL3Qi5wpyNybvkXKxGJS04UEyEm1umkq0LFQaHsoFll+QshomUnHJoqXktvWcjOTsjwFdFbqsB1k8g/fH13L1rHX/dJclPwGe98uYwFQsJtcNlnlBV2MzAYMJhBFBeeJ0JFuPmMhWLFRV/Jicgzsyc1a6tl6mQf2aIGAkK/4JfBIjM6yz9uQqAiXJtb/skrWnnts1DPgFDaGrJwKOV971GRvyT76OQlZnGL9ZE4KWsu7oePLsPP/VLHpYqrHVgKZRfFsotGjezD7ul5UnE98VrEsFFZ9hELxcnoLTme5R8n+0gx/DcKEVtiZVKR4NkMNWsz8g8+z3xRoy7Jy8BgoGDTYfa8hZNnFJo/T/IK9jELvrdciWT+Zi0LbQ1Z5bl1ln9G8aQ6NFr0S2f5xzlsVVILs9ZZf1QDlRxBYlskFOff2pABpYFDsqcYHQhl2UdnxbP+i5Z8JcHy/6EQWVRy3FjZp+yGq6iK/dGt4xsi/xiHL6X6wbe35C1NmU+w/HWDurgmgSraJ2d7so/qOx5V5R0MDGoB9kiwBziUSauSfZSWv0ckvaWo5k8IoHvWRfIXX6tkgaeWbcWCbzwKADjzoGkA9JZ/HPmrrG0mCbDzqshDjnyxCAnJU/mMjR09JV6jSG35y5q/up/sXGKJDFGOSUIcgRZ8v4zqHqtCLxmWbgyK8RWkctziwKgq08zQW6r4lr+eiMuOftbSK804gtdM9iGwCFF+x0bzNzCQwB4KlwLrO/pw2g+f4vvUDl+17AN4coD84BOiD7MT4/zPOnAaf63S/FWfcynFss1dmH3Z/VixtZvvj/NLqAY0NtPI2WHLXyY1Bpd6lqQoT1kWwbMrduBvfuRTTylK/mlWDWPLPgLhBXHERWFU2GNiU+g8ugGwp+hLJIrZlWplLYafPbUCn/rtIgBhx2s5Eu3jLe6i0th7S04oEU4FMZFLRp8g+6gtfzui+TP0Y8XPVDDkbzBiwZQLCopfP7MSb20JSLSkCvVUcAMj416F5m8Rok2tz9mB9CAeo9L8VeerOBT3vORVMX9wySa+v1rZh51PDvUsKsIJAW9QsCTNXx70uouOdtHzPSc1a/siWuyikzwp2mfm+ID8Wf0bFXp8ArUVA6wX4aO/dywijMk+FokOGK4fIqoawJnsE7f2smf5q8lftPyVGb4ZAkKI8h4Z2cfAQAIRLH/5mUtb24dr/gprV3aMqj6XsUjIIZdk+TPL0aGUy1Y6K12GihSZlZrnoZ7edYtkIxIKW/dWjPbpKYWjmeJkn99/6h04aZ/Jyr7oF3hhsk+8Jp/LWP66usFx7U2Bz4VJN6rZWEXI8I0DI//WhmxE83ep911krSgtetFLbmKoZ1rNn12LqPkTqO+RkX0MDCTY3OEbDdFLHe3jOw/7Sk4k/j/W8vfJP2tbIZkgjhwALy7ftkioQqTY9VjNX3FNuiQvMbqkIsk+ssNXXmaxp6iK8/fazdiWV0ROafkHMeti/+TyDtHPURw1ZwL2mdrqRfsIx01ozgn98ghUJcvIWb06sCUcxzVmUZLKOzi+7JPRWf7Um+Wpzs++y//3ZLR0BRCOMrrxieXY/8qHsLWrGCR5ZSxYRP0dG9nHwEACewZVpK4kfx4VFGxjFnOvQvO3YjT/DHe0kpBlJlfLjHzOJj75q/fHlRdQ68FSkpfjoqtQxrH/9zg/Jpzw5IV6ijMU0TkLeDX/RbYAACAASURBVNaxzvLPxZA/I3m21gEANGRsweEb/syGjj7sfcWDWLyuA1nbQi5jYX1HH15dt4sfM6FJIH8m+6giqly95i9fG+BFOD3y+mY89FogubmUyT4Ky79UgeNS2JrfRKNvRPzqmVXK8/YKjubXNniLAW3uLHB5LsujfaLXUC/L38T5G4xYMNnnz4s3RhbPVpITC/UUbPx81nf4KjJ8AbWVB4DHmp+7YGZIYsrFRIMAHnHZxLP8M5bt90foY5WWPxuEWPhjb8mJZCSLsg8r7yD6JuRTVhwK25LIv8Isfy9SSGVls20t+Qy2+QvbNAjlGmT56YFXN3r3zvHuS9YmWLa5G8s2B76b8SHLvxK6ZhGOG2/5sz19ZSek2//kb8v5a1bPX0X+X/3Ta7AIsGD2eO83IcUTNGTtkLQjo6+kLmsdkn00Dl+j+RsYSGAW0aLVO7FmR29oX5zsIxIse7BUmr9F9FZXcy6DV686Bf97xn7VWf4WQca3/KuVfZQOX58IW3xru6dYiQxY0VBPvS+DnUdXBjljeQOHau0A1vdWITeiIWsJ1TaDzxTKDrYLBfMyFkEuE43fHy9o/qxLKinOK++g1/zZrM5bWN7C6u29kWM8zT8s+3z7Xw8K7beFctjv3Gsi/vXwGd51JhT6Uw0MXuVT1/MbWb7DV/H9m8JuBqMWX7jrZXxXsXhJEuIyH29+ZmUkNl2VD8CISRXtQxR174N9HsnJGrBKLxZh+w+647qBwxdhi1gHlRObna/Fr3fTVahEjhNJkTt8Y25exY2uF1woO8jaHkFlbUuZrSta/gwNGVtw+AafufXZ1dguJL7ZFuFZyiJaFUl2KjJ0EqJ9xOtozNnKkh2q+j3nLpiJH513SKif7NZNbMlz/0aDIutYhIr8e0oVlCoub0P3lZgMX4NRiz++uB7XP748+UAJcc/6xl0FPLVsa2hbsNh3lGz7FEleFtHXVRGtffGYuFBAwNN2M5Zv4fltVBwa1PmPjfOP7mMzDUa43cVKZNYjkiKVnJb7TG2NtFl23EgbxXIQ6ZK11ZmorBRzi6j5Z20+ixBln+seXIp1O4OaTBmbKLOmWxuiyrTS4Zug+bM9vSWH6/NA2KHMAgdk2WevyUGlVvHeZSzCZxQNilmLiD4V+Re9yqfsugl0hoax/A3GEO5auBbzv/bX2Kl8UmhfpxTFopJ92Ov7F2+MlGAgMZq/uDkk+6Sw/G2LhCSm6x9fjsOvYevExss+k1pyePbydwfn8ztiWwRNORvdBQX5S0XOLOIRyt2ffgfuuOjoyHkqDo0kdfWUKjxHQbVkJKVB3oJI2F6VTnWS19qdgfRiW5Zy4FTV7tc5fGM1f39XX9lBQ9bG+UfNAhCW0hxK/QzicPvid+rJPhbvB2u3IZsg+5SjM43eooOSEwyqOsXQWP4GYwpfv+819JYc5fScQVco7BvvOwAAsEuKYmGHq8i/q1jBy2s7QsfHZfiGLf9gexrN37aiyTxsndsk8vdko2CbSIQt+UzI8v+vd8/zPueT7ru+9wQeXLKJ9/3wPSaEHKoMjksjkpmnlXvWbcYmkXv//Mod+MGjywAArYLsk89a2vIOIcvfIkpH67jGLN+vumaGiku1i8CIYLLPN//lQBw8Y1xokNvZU1bG8svnZrttEgziqmJzInqLatmnWHH5oKez8OtV1dOQv8GwBCMoFfnf+9J6fP7Ol7UP+/imHAgBdvWGV+Dii32kTKqyJKIVQTSWf9IKX8zyd6jaQZkU7WMTEhqQxKidloYMuoqB5s9CLlmbLApIJpmJ0gBQdt1Yyz+nCPUUNW1Rp89nxGgfebYQvLYtteyTy1hYfu3p+NKp+4SOlZFk+QPA429uQZ8g+2TscMXW+1/diLU7+jjZs9tkCz8CS7j/tk24QZEk+6gWsWdLXQayjxomyctgTIFFdMgJSABw6R8W456X1kcsdYaMTdCazygsf9/hK1r+lOJd+05RtkOgJhpCwgQajvaJf1CZ5e+4NHXxOYbuYgV2JK9AsvwF2YctMi8TtdzFp798Ei45aS5/T2l0ScveksOL4GX9JSPFdkMzEEH2YZa761LtTI1dhy5BLmNboUFOH+oZf50f/dUL6CpUOPnrsrFZP9h5wpZ/8NvMCPJd0qDPvpPxUsZyqRKEnupI3qzkZZAIx6UoVvSxxiMJ7HlTWf7HzZsEAHhzc5fysxmLoL0ppyV/0fL3YtqJUle1CFE+kPK2cLRPkuVvcfKXSblQdmIdvtu6i8hYllRLSCH7OIz8g8XGdf31jsuEyiio0FtyeE4E07fF2YF4S9p9wh/XmMWcSV7dHjEUU4yeEfsUN/CJBKwjyaK05kBeYY13Fco8Mkc/2AR+FPGv99rifRHX3I0jf1EGWzB7An8tW/46u2FYyj6EkA8SQl4jhLiEkAXSvssJIcsJIW8SQk4Vtp/mb1tOCLlsIOc3COPGx5fjnOufGepu1ATsB9+tsPxVy/iJsCyCcY1ZdEjkz7g2XOWShmK3RRBNtI+8KUT+aSx/f7m+kiRbdfSW+azk+L0nY45fRI1hW3eJy0ZBe4LsE7H8PZKT/QsqbTmp345LuezDSLUo+AVE4p3e3ogfnXcInvzSiVwycdxgdS1VBE/GIsqZ0Dv2mujtt9UDngjZ8FFdU1ehgqZsPPmz7VzeSWP5xwz6zf7v1bYIDpnZzrf3+P4ZNgPRaf7DVfZZAuD9AJ4SNxJC5gM4D8D+AE4DcCMhxCaE2ABuAHA6gPkAPuQfa1ADrN7Ri02dhaHuRmp0Fcq46Sl1LRSmgIqLgjAkRflkfPKPWP5uVPapuBS2H78e6YEmGUo+1hJIIslI45q/wvLf1VfmA9Onjt8T1/qO69DnI5p/WG4RHb6MdJJkHwCwUxSQYaTPLP9CxcWuXu8ei7OAjEVwziG7o70px++fWHVTGbtvWZH8hFXXnYmpbQ28zaD/UjSOvy+ydKfiOruKFe6cFWWfIwWLnG1nJB8if1Hzt9JZ/kwGa8ra+PBRs3Ds3EmwSDTOX8fxw1L2oZS+QSlVZeecA+AOSmmRUroSwHIAR/r/llNKV1BKSwDu8I81qAH6yk6q+ibDBY+/uRXffGCpcl+c7JO0KIhNCMY1ZTkxMYiyT7Hi4LQfPoUVW3tgEzVpE+hkH+l8QvRHUkx2UNsnSv4dvSX+/bGMT4asIEWITmiRFFsl2adRI/uorinJ8geCctQs6ue5Fdtx8NUP469LNoWig8KWsvfadQOHr9Lyt9ULmQTtiJp/mLbY9cjkr7uiBsHhy3D5GfsGnyNhrT808AgzLzFkVyUxMbAcjIacjfamHH77iaNw4O7j0COFeuri/EdabZ/dAYgLaa7ztwHAWmn7UaoGCCEXAbgIAGbNmlWHLo4+FPya4yMFcTH87AFTOXyTBjhbY/k73OELbOksYukmz2fglWVWk7xa9lFb/paVrM96FTW9aB9Z5ujoK3OpxrZI6Luc2JzHps4CMpLDV3a0hix/5vCNkL+6X0lgmj8bBJ5Z7tXIv/QPr2BSa54fp5JoKq7L+9GikO1sjezD2wkRcHgfux2y7KMbiJlEI0o1ogTEru8IfzYQsfxDSV5+mzGWf3Pe+06bhHDQplwGvb7lnx+umj8h5FFCyBLFv7pa7JTSmyilCyilCyZPnlzPU40aFCrREgXDGXEkziwqleafFNJnWwTtPvmH1q8VQj13CHVlLIso4/mJpqSzzuGrm0GIYA7b5Vu6sXJbuADbJ29dhFV+zRlWBoJhYkuOb7dDMwJR88/CcSk6esuwiLhwjCz79Nfy98iLlcFmklxnoRIqJieHRgLA1u4iFq7agYxFeBSSfP5Y8pcSrVTXIzt8dQazmKmsar+tIYu/fPZY/Oi8Q/2+hVc9Cwb75GifWz52JBqz3vWKmcXN+Qx6il5tn2xCtE+9krwSLX9K6cn9aHc9gJnC+xn+NsRsNxgg+krx0SLDDXHkz+LGu/qh+dsWQVtjFhWXorfkcO1bTPLa0RsuKqZ67iyiTvKSNzHyF4lBB1bYTUxwEvH3t7ySFDYhoESw/FvyfLs4KIhW4b67eaUa7n5xHXKZYJ0BebBU9XG/aW0AvEiqp9/apuwbl338QWCNojgau0b59Xk3PYuO3jLyGStEguJ1sLDQxDZtmfy9v2llH7YOryj72H69o4pf++iA3ceF+ia+ZrWYRIdvXkH+TTkbJ+w9Gb9f6IkdYiJYc96rL0QIUmj+w9Phq8N9AM4jhOQJIXMAzAPwPIAXAMwjhMwhhOTgOYXvq1Mfxhz6ytFqjMMZuoGKUsrL96qSY3TLATJ4RcKC+vaq823vDlv+KgtfV9hNJk/2cNopyF+O05fBumhLmv+k5sDyFyGuOnXiPpNx4O7jsKuvzOvuA9H7pUpc229aG17+2nvwb0fMjO70waN9fPnnbWHtYREqzb/D97+UHVdJlBmL4Jr3HRBK5gq3GZ1N8Pf+OeSsZB1Uso84y5O/8lCcvxDe6Vn+4TZFnLiPp1iwwU4c9Nobs9jZW0I5TYbvcCR/Qsi/EELWAXgHgPsJIQ8BAKX0NQB3AXgdwF8BXEwpdSilFQCXAHgIwBsA7vKPNagBCopFOGqBXz+zErMvuz92Wt4fyJY/k2iKlWBh7f5q/vLKVkA4ykes/6/T/FOHejLyJ/qMYPFcqlBFlmTFpBQ5x2DG+EYAiph9oS1CCOb7FnwuY/PzyDMl3eDjRefoL4DJPczy18lvcVY6Ww1Lhm1ZGNeYxUXH75ncptRHtidi+WuuUyX7MF8Mey1C7G8ox0LQ/OVTPfz54/H9c718BuZgFjX/iS15dBUq6Ck5/Leqk3eGZagnpfQeSukMSmmeUjqVUnqqsO9aSulelNJ9KKUPCtsfoJTu7e+7diDnNwijzy9LLK9IpcKhVz+Mq+5LN+5+/5FlvP1aQrZIGbH1CNY+0/wffm0TfvXMSu9zKciflQeWi5oxiEXcLI1WL6byy9vD5/O3p7H8NccctafnXNzV5y9YYof7tLcv6aySpBZZq29r9CSufCZIRpKjo+JkhLg1iINon+QSFgxJ94O1GayMFp94Bagcvupz6M7M11+2xdlE0O+4+yPeHtuy+LMmP3JzJ7dw0mdyT4OimiibpXnnVZ9zWIZ6GgRwXIq1O9Qa6GCBLVGXRvnZ2VvGr/+xKlW7/Iddgx/hss1d+PRvF0UWzwaAw655BFs6C6E6MWzpvotuXYSv//l1f4Hv+AsUSwWEZB/BMBTJX2v5Q22lauP8NRnB4b5ZymPGNWaRsy1ej0i2/FnpZbnyqEz+LIaekIDcrv7L63hxzU5+TFzh0bgM5UD2ia9jI34/SYXuxOimOOgS2+I+m+zwDUtJQeZuun7YJDAo9pzcjE8Ksxbxd8MWehFlH7GeUpDhq5F9hiraxyAdfvjoMhz37ceHdABg5J9EjtWWgOBaeQ0UpS/9YTEeXLIJr23YFbHgOwsVPPLGZk74QHRFrjc2dipryYsQlykUj9VZ/jpr3CJBBcfw9vB7Me47jeWv6n9j1kZLQ4aXobYlJ/RsKdtXPjcDK+ZWrLihgeHLf1gs9D/G8o8hGp7kJWj2H3nnbOw5Odw3UX5JS+pJpbBFwpe/E/0pdLKPtz0XkswCv0Jcnz1r339tW6HVxS4/Yz/lZxoUA5y4jkBSbZ9hKfsYBHjiTS9KQwwhHEyIhcJki3rxuo6QFCTHv6dpG4hfaKRaUKgHqaxtoacYLLJdctxQ319YtTOF5W/xB1tcACVE/l3B96TLzCVE7WzThXp6cf6xXUPGInyQFtGYs3ltHnaceB5mpR46qz30OXkW0uZHzBTKToj8xWuPkzVEy/+mCw7Hdz94MH8f1PYJLNiDZozDV84MSO8Tx87hfgcgjdXqk3/CDEGur3Pjhw/D/tO982hlH82pGdmGi8UFMlms7CMl2LH7yr6rD/rLOopgPhKxWRa6CyQXhatXqKch/yqwvbuIC29+PrT8HANfiDnhi6wXxEgH8UF/ZW0H3nv9M6EKmJ1Vkj9rrhbZw+x3TKm6vZxtcc1/fFMOpYobGqyWbeoKfe7k/aZg0VfC0ciWBY3sIzh8JctfV95BHecvvw9kn6SwPNsiKJTVln+zkPyk8h8sveY0/P6T74htv82XfYplN0SY4p2OIxPRAj9sj/F8jVpAkH2E3/i4xmxowZWvnDU/4hSVP6M8r/CZfzl0d1x++r7a/TYhOOPAafj3o/eIvR7dZapkn+a8zduJuz+hBDsx8sff/O1/PQirrjszfD5+7aLlHyTFJcX5j7RQz1GJW/65Gk8u24rf/HN1ZB+LKhF/UKu392DTrsGptSOSv0hyW/yoFpFAq7b8uVOrlpFE6vrrWdtCb4mRfxaliosNHcE97ClVQp8jhGBiSz70wGUsIczRdbFiazeu/vPrITlCnKHJJRMYdA5f+WHklr/meBEZmyhDEhtzdqj6o62QnBqydmLVUGb5lxw31M/wovX6z4uhoznJwctkH7EPbY3Z0KAlQ1UZUwTri7j/B/92CD55wl6h40JJXqz2DifrKjV/vi5BcEBzLsPvV5zM4sX5B6+Z5DW+KeefU/9ZcVe7kNMwVFU961XeYcwh0KaDh+yE7zwBABFLoB7oC1n+wfbuokf0YtRLR2915K8qhdxfsAeAUihzErI24eGd7U05rNnRiw0dQUJUbylcv0g1exAt/1KF4uLfvYQ3NnZq+yQXS+N9RTiUUmw/9F7I+EwO9bRC1TAZGjI2LwMA6GcjSWDRPjLCUU8x/ROul8kjbQ1ZFMpFpfXe1pCNjf5hFrs84F1//qHoKVbwvYeXRc6rbieckAUgFVmrwDT/ULSP4GOJ1/xJ6PUVZ+6H98yfGkoKk8GMJrFVcXbEBiHdZRjZZ5iDWZUJvsi6IST7CE93t6+fiwtdiJb/q+t2adtcsn4XlqzfVVPZh6Hs6FdeYpZ/u2/5s0qle0xsQk+xgorrcsJR9SljWcj5WZybOvtiiR/QJ16lDfVkD2eqwm4WQUGYgezW1oCnLz0JlkXQIlS7TOM8VqFNUTETCDth42ZwWQX5s8zbvILk2xozsZY/Izn5azrroOn4tyNmpSJceT8bBywST5q6Qmk5heyTpj3xGNanfMbGcfPiy8/ocgE+f/LeAICZE5oibevOWUsYy78KcL1asa9USY606SyU0ZRi6t4f9JXUmawsTl6M9RbJ/+zr/46nLz2J/wBFnPWTv4feJ1RVSAV2D0uOqyzT4LgUPX6o5/imHEqOywe2yS15bvk35zIolEvK++2l6vtZqFu8mjMTm3PYrnHGe6St6CtJV9hNLO+QJPvYFgnlS1BQfu9bRMtfk3uQBFXFTADY0RP4OOIXTQlbw0AgJeXsaIjnuMZsLDmlqRmU5jhVkpeVMHAkh3qG5SN2fKLsQ4PyDmkQWP7h4//75Hm4+KS9gv7rLH8T6jk0eHbF9lDSkQ5M89dl2FJKcdBVD+PSuxcr9w8UfRqHL5N9RMdnpNplSou+FtnDzDL+8WNv4a+vbYrsL7teaQdCPNIpO5Rbre1NOa75N+XV5YoBz5JnDzjL5NWFSgJ6K5to4vblTUHd92QrLWMTFIRQW9Eibwk5fPtn8TUriqYBYcs7rjqGSn5hlr8ySilrxzpzZWK++SOhNZ84IdoJepmqsBu31DWf0W1nGjv7jTBfBjs+bgAXv5O0pHzIrPEAghXoRGRsK1G+MrLPIKKv5OCel9Zhe3cR5930LP7r9pcACA+9ggSZZa0jUiZx/PHF+tSxC5G/YFAzy7+isfyB9OH7NdH8/b+LVu/E2h3R4maO66Kn6KA5l+GkwmSgcY1Z9BYdVBzKSU5n+TPZh8Xz76GY2TDpSF4dK9RWGstfTPJKeKLEBUDmTmnB94RQSlE+yVhWvzI70xBSvOwTvQBWME4VyUYEqUtVsE2cSZx50DS8a9+p0ufZcVVo/pJGXr3mL5F/NhxtE9dcNau2MRwysx1Lvn4qTtl/t9jjBru8g5F9FLj6L6/j9ufX4PrzvZKur8To4gxJsfBJiUkDhSglOJRiS1cBb2/p4Zp/WaP5A8lVMhlqUTQu6Xdcdih6SxU05WyuzfYUHdgWQWtDxrf83WChEsX9toUMXxbttMdEz/Kf0JzjkT6TWvJYt7PPr+qZnvzlLdVV9QxI7KHPHR+ux18Dyx8AfvyhQ7HXZP1MJ24GpyK0z79nb+w/fRyOV1iuDHdcdLRSOhQHQ1XxM25tV6H5s8F2z0ktAIAj50zACqk8NpBcz5/NJvjMJZXsE7yuRo5RrWEgg82CcnZ4VbM6cf/Ytfwv/+NiPP7mFuW+9X50CbOamZ7Pvhz50fnF0yv4ax1Bsmigek3hZIfvB376D3zo588Gso8gL8g18tOOS4NRMJRp/s35DLc0u4sVZG2C5ryNvpJn+ccVFxPJf2tXEYQEhdHEEDtWJtnSJHmxtmREZB/R8k+h+e8xsUnZdrMU6tlf8n/vwdOx/3R99Ems5q8g6Kxt4cyDpsU6s4/ecyJ2b2+MtieGjsb4uqrR/Nl9O3DGODz1pZPw8WPnxH5WBivpnJNkH3E5Th3E7ySt5Z8WrDlZRhuWVT1HKiiluP35tfjor16I7Lv12dV4ZrlXz5xZlUyX1X0H37j/Df5a92Ax8h9ozO7LazuU03ZZ82eSCgvrFElS1m7Tav4Difb5x/Jt+MviDdoIDIaK46K36Fn+WW75V5C1LTTlMqi4XpQQGxjEwZatw2qRwKrb2l1EW0MW45s90hdLRDf7swddbR8gePD2m9aGE/ZWR3WEHL4pMnzv/vQ78aeLj4nsE6s+plkPOA1WfuuMiM4e9zXGFXYTcfen34m7EhLOgPDvnZGuCJKCcAFJ8xeOnTWxSTlgnXXQNPz03w/j7w+eEQyGsuzTwGUfhP6qIMp2tY6/Z/dCjqoy5R1qCJHEZCL96r1L+H5mLctri4ofkWvP6GQf1sZAsvUeX7oF77vhGdz23JrIvpDsI1wfi3ARo30KZQcHCnHJaR25A3H4nv+L53DJ715KPK7iUvSUKhHLP2dbIXJk1pF4rTd/9Ag89LnjQUhQz99xKdqbstxp2VOs4Bf/sQAXn7RXEJ8fE1nDpvZ7TGjCkXMmKI+pZg1f2yKY1JLHwTPbI/vEFa5IirbSgBAS0fHj5LukMgsMh+8xXns/wu0J5B9n+VcR5y9/VyoL/PrzD8NBM4J7/KdLjo30g52TL6YCpvnHR/vw1zUmZdacPEMy5F9DiFawbkUlIFg8PI7zOvrC4YO8Bpr0oVrIPqu2e7rmW5u7IvvECBKXBhYcc3iKPofekoOpbXn88kLPIqy15e+4FPe8tE5NMgnXX3Go7/C1+UPZW3KQy1ihSJacgvxb8hns4zsnRaJpb8px8u8tOTh5/lR86dR9ObFnbPViLkBQAZOCxiThBNZrmpW8dGjOhR2mtTIsZUKPc9zXWsoQyVKp+RN2XPraProM67QQDQNAiPbhfYkhf1H2STlLSgv225Ed6ynH4+rPV59mhzdES365ZjUiQL2QiAw5W5bPGgRLu7dUwfIt3nnSjOI7e0pKaSduECqUwrIP+4Fz2Ue45r6yg4aszQkvbRRPWsv/tudW4/N3voLfPR+doSSBWf5N+QzPfOSyjxAHP3dKC2ZOaMQVZ6orKYbIvzHLY9VFiOUB0pTTTUrCSRvnr0OT5BSslcUnSzlxY3itY8rttJZ/FZp/tZ+Vwe4HW/OXyT5ByKX+s+IyjkkDVrVg55VlLBPtU0OIYY/lmNWpOqX1Y9lXQAWX704pcYgRqZhR+4lbFuIfb28HkDxV7CyUceg1j+BTJ+yFy6TiVrwfijb6pNo+uYzFk6UAL36eoVBy0Ji1+UMjW+irt/fg/J8/FzlHWvLf5kfYyLXnASTGlXqaf9jy7y5WkJcs/5Z8Bk9f+i5tOyLhtTdllZmv7Lvwauyr2wmt4pTgFLZJcmRGnKTQFLH8a/PQywNObWs0xSNJ9kmb4Rs3KImf/fJp+3KHOuCtkNYo3Vd2/G7jPIf/8X6GLmsl7jsKrZ9cc9nHa0+XeVxrjFHyD4hZtNArkrbfKVn+Kotpp2T5u9xfEBzMiB9IJgdGnP/vybc5+RcrDnYl1ONRkb8I2fJvzNn8xytHzNz6z9U84klE6mhVv10Vx5QTwkq55Z/L8IzSnmIFLfmGiEM0vgsEWZug7FC0N2ZDZYjFY7y2ktdPpVRfLkCUfQbiBJTJn8QYlh88fIbymlSQyaSWZTqSkOjw5SWd+3/fRFlL9kV8UbEmMPuu505pxdOXnsQjwVgz6mJ+3m9ArORZe4ev91d+dusV6jkmyV+0gpmFvmT9Lnzv4TdDx8myD4uHF5+djl7J8mfkryG5pGm1OJi4LoVlEXzhzldw/6sbQ3XTZYjlHSiN/oDEQa6v7Fn+lsby1/3Y0lr+7BJ/9NhbeG1DJ35xYRBtkrQOcNlx0Vty0JwPLP+eoq/5h5Kgkp8Ib+EUqpR8vP3RPlsk/P2GtWb/rzQIiMv/DcRKk7Nz49r6jpAclgTVOrqDhbSa/0DkJrH4nlgcLw3E3IRAvoseR+BNWlWL09cK7Px5e3DIf0xq/qLUw0jxrJ/8HY/7C7IwdEmyD7OQRSs6YvlTpvlryD/hmxQHE5a4wsogyFFHIkIlnQXNn4H1x3UpCmUvSYr9eGXNX2cFx0WJbO0q4op7XkWh7ITI8dE3NoeO69WsA/z+Q3cHAGzaVYDjUkxta+DSTclxI9E+afRWNtAwyefWjx+JR79wPN/PvgvH1ZceVmn+8u0Ro30GwgdN+cFx+NYiWS8txFlHnOY/kEFTNATSJFPpwFrRlfrQ7asVWMtiHsLNH1nAHdK1xpgkf1GPrzguPneHOgQxavlHnblRy5+1GxwjPsRJPx5xMGHRRozI2SIgqiZk2Uf+wbD+sKigxqzN+/L5O1/Gz58KEtV0PYxzDH/hrpdx23NrsHDVsqRhuQAAIABJREFUzlhLRR5QGT64YCaaczbe9ge8mROaQrMXFufPkCbSgnEcK3Z23LzJmDulle8PKk7SUNiniDD5q88TivYZAGPXK8Qv6vBNJn9VqYb+QBywlZq//3cgfgjxO4qrMJoIPrjH+xdo+PCagUf72Ba++8GD8dj/nBAph1HT89Wt5WEMkbyLFRf3vrxBeZxIVJRS/jlx8Ngpk7//I9ZZ6TpuoJSis1AODSby6mCsDjyzrNds78VV970Gx6XhSpE0qvkzy58dJ1r+27pLuPaBIFFNx/5xBuNzK3b47Vqx0Zy6CCqml6/0o6/2mNAUynTMZsLRPtVoxDrZhz1sLqXaZKM0hbyqKe8QB5l0akUu8jUlkf8jnz8eT116Uk3OLV5TXALZQOYiokY/EMuf3Sbd2g5AWCqs9RxAdH7/6+EzlOUyaokxSf6iVb6ly6sV/5750RG2sy8gqkI5KEEsfn5nbxn77taK75/rabDMgtEtnqEjh9ufX4uDrno4tNwiI/8st/zDkslnb38Rv/7HKizd1Im+shPIOAqHL/NzsBlCQ9bWRivoHJs6ucBxKR/sihU3lrSKGs3ftrzr7Cx4FT13H98YshRzNkFTNr3DV4Suxj1rwrP8/XZjLH+dRRiUd9BHgDRmbZ5XkRa1s/wl2SeBaedNbcXk1nz8QVWAWf+qwnC1KG8uDspJy0XGthMT6sl9E/WUfVJmO9cKA7rzhJDvEEKWEkIWE0LuIYS0C/suJ4QsJ4S8SQg5Vdh+mr9tOSHksoGcv78QrXK2ROAsxSgrSim9pQonUDE6Znt3EZNa8jjML9saSEPVWf5PLfP8Day0hNgG+0HLa78yK9pbF9bhSUIOpZGHgPkp2ADSmLW1PzLd71sXJSLOfsoO7ZdD0SJBpMz0cY3IZ+yI7JOxLX5daTNRAX2N+5DmL8Tqh49RvxYhWv66ezdvagvevV91U/haEY08S6pFae5qwAZflexz0wWH42PHzMGcifpCdNVgIFnR7JPK9ZwREHO9QmW5ATISyB/AIwAOoJQeBGAZgMsBgBAyH8B5APYHcBqAGwkhNiHEBnADgNMBzAfwIf/YQYXosGUhjWJssAq9JQcOk30cF6f84Emv7HNPCRNbciGrG4ghf80XyzJQRc1/XUcfdvSU+HSZ6fXst8mIvFD2FjxhU15Kw/pxa0OGS1bM4RpL/pp7oC1dIVjzpYqbGNGjgliMjRUIyymchUzTrcryr0L2icpJ3nsKPRmHCrtVeU/jUDOH7xCGegLB0pKqaJ89J7fga2fPr3lyWX+QKsPXL88BxDuw+3V+fo7BEWQGdBZK6cOUUqaNPAtghv/6HAB3UEqLlNKVAJYDONL/t5xSuoJSWgJwh39s3UEpBaUUa3f04u4X1/HtbH3YJH2tp1Th4ZvLt3Zj2eZuXHHPEmzrKmJicz7kPATU1SYB/Q9rXFNAUGwg+Oq9S3DYNY9oZR8mofSWKugrOzxDVJZ9xjflIpp/U87WFiHTGU86i6cUip5yY6OSdBAtf2apZ4VrYNfTJBRjSwu95e/9pYLsI5MQ4cfEOHxZtE+KDN9qUIvaPkD0Xg2y4c8t/1qXQ6g1YjN8hYHhhg8fhm+9/8DYBYL6A74o0CDdplrG+X8MwJ3+693hDQYM6/xtALBW2n6UqjFCyEUALgKAWbNmDbhz77vhGdgWwZodfaHM0427/PVhBfK/4oz98NzKHaEwRU/z956aJeu9NWH33a0VL67pwKTWHH/oGe/pMod11mObQFBT2/KhmvtZOdqH98kj8r6Sg76Sg93GeRazbKGPb8pyJzXX/HN21bKCjtNFsldZ/mlCC22LcHJoYBqxyvL3I35UC6vroCP/0w/cDb9ftA6HzByPx5d6sptM3uK7pESwuNLQQ4lonP9gW/4e+ZfjlhAbBLzytVNiI9bYXVKGevp/beJZ/h86cuCcFDmHRnqsFxLJnxDyKADVEjRXUEr/5B9zBYAKgNtq1TFK6U0AbgKABQsWDPhXo1uQxXEpWvMZTGjO8W3HzJ2EpZvCxdMqjhty4gKBBDGpOc+tP/bjKmsILw3fzpnUjGWbg5pDzOqVSzEXuOXvoFB2+RqwlFKhaJWFhqzNH7xUmr9GpEgl+zhuxKmbZiZgW4RbqKxWf05y+ALgqfrVWP66OOl37TsVK791hpecJVjvOiTV/0lT1TMJt378yEjuyECRlQu7Dbbs4w++nX21va5qIc6uVbBiyLcWyWhJYC3XurieDonkTyk9OW4/IeQjAM4C8G4a6ALrAcwUDpvhb0PM9iHD5NZ8KFU+n7UiqfZlh4ZCPAFgS6c3g5jYElj+QXkHNeHpvtaSMLAcNKMdD70WzDoYCTLJhv0Q2UPcXayg5Lg8Dt5xg4iOFr80MssZ6CtH4/wjfaxW9pEs/2JFkqfKacnfu87GnF9f3QpKNASaPyP/2uiicvJOPPlrtgsJYtrPpxwUjpunXjNgIJAJa7At/9MO2A33vrwBe09tTT54CMF+UmrVJxjg64W0dY5qhYFG+5wG4FIA76WU9gq77gNwHiEkTwiZA2AegOcBvABgHiFkDiEkB88pfN9A+pAGslYuo7UhE4qOydlWtDaOG7X8N3V6ktHElnzE4SsPFAw6o0t0EB88I1zrnZc5KAWhpyIRs2UJebSPS/kD3tKQQcYivO+9ijh/GTrLMGmhGnYdsuwjDwYq2CSQfcQEIzbwZbnm7w1w9Sqn2y/LXxhAhqPsI4NFpg0WTjtgGl696hQcOEO/ulgtIM7e+4O4hYYGw/JnhmM9w0lFDFTzvx5AHsAjvgX1LKX0U5TS1wghdwF4HZ4cdDGl1AEAQsglAB4CYAO4mVL62gD7kIgdUuVNGfmsHZqu57NWJFSy7LgRJy7T5Sc25yIO35JG32SDwktrdmLG+CYeTy1GIIkLrQBBckyPsAqVWLFze7dP/r4M5VKB/PMZZGwLZcfFI69vxlu+nNSQtXnSWLSP/Sd/leavK+kgQrSaxVlYe1MOPaU+PgiwAS6NdXT9+YcmfvcMyWF2+nr+QWmIwdNr+4vvffBgnHHgtEE/b6sm16JWePrSk7T5HGnBnfuqff7fen697PkaNrJPHCilc2P2XQvgWsX2BwA8MJDzpkWx4uBLv1+MY2MWngaiiSH5jB3RicsO1ZLfuKZsKGwQiFYIZWAW+L/c+A9Mbcvjuf89mbcPACfsPZmHxjEEyxl6JOq44VLSLCuYSVUe+Xv7WvIZ5GwLFZfiP3+zkH+mKWdrw1F123VyQVK0T3cxnNX7sWPmYOOuPjy4ZBPfZlmE91kk/3GNWazv6AuiffwBLs0DctZB0xOP4ecXwjWTjtFtj8vwHS5DwgG7j4uUOB4pmD+tTTujrkU2LDMAVfIm3zfgs+jBjK5h4/AdydjSWcQLq3bgvlfU5RsashYKZTdC9PmMQvZxqJYUc7bFw+d4tI+O/IUBZHNnEHVUclxMasnjlo8dGfkMkzwYibqUhqKB2GtGmo5L+RTy4pPm4u4X14UIOuPH1OuITpa3GHSSVVK0T49E/vtOa8XcKS0h8rcJ4QOmTP5AMPupxvKvBrpIC3FGmGj5jwDZp8ah6YOKB/77uLq2z746lY1z44cPw8+eehstufpRJjOu6ulXEDGCfwrJmDmhCTddEKTUz5vSEtrPtOUGacHknB2VfSquq7X8s7bFnUVbugqYfdn9+JOmXpBIzCLKFZdHtMiQk2Mqrhuyphn5sz4z2eeI2eNx/N6TkbEs9Ar+AnbdOgtDZ12lkn0cGiX/Upj8c7YV0ewtKxh0RM2fzYLY7Idr/jVOhOGVOWOa1Vv+wf7B0mv7i+Hev6GEWO5DxvF7T8Ztnzi6rlY5s6FGhMN3JGD/6W389R5SCjkjEtnytywSsfxLFVcZvsm0ajZaszIN4gIuIiqaJKiy44aSmuLguDRUIC0gf1/2ccNZq1mbhI5vSLCe9ZZ/Ss3fkS3/sOY/d0pLpMiXTQhPomPRPoCQIOT3lUX71KuWus7qikvyIoTgo8fMxon7TB725DpYxDISYXHZZ2jOz2qHGdmnRhBv5EQpGoBZ/PlslHSjlj/lX44IRmLsoXp7q1eSuCWfiWjdrB01+VNturhscVcciu4ik3osvtwkux6HUrhuELqWta1Q7D2zrLWyj8bCF2csBb+QXNa2QtdTdtxIaGevZPnvP72NL0bPkLEsLpU1ZETLP1h4HQAac+k1/2qQJswuLob/yrP3B6CfHQ2XMWG4D05DCRJj+Q8GuOVvZJ/a4c6LjsYXT9k7Ys0zy79BkQSkWgZRZREzwiYkXNRLlpJ4O25UFgE83VxH/rL/QLT8p7Q2BJa/T+o7ekoouy4nMlliYY5hnXKi81eIl3/uz/6J7/orn7HrachaSsu/W7D8D9i9DYSQiGwjyj4Nuajmzwa4A3cfh3lTWmpaddKDOtSTnX96e2Mq4tTODgbWuZrBWP5xqL9TNw4sidLIPjXEUXtOxCXvmhchwXjLX5XkFf1ZiHq8OGLLFTgZHJcqyxqXHb3mL+cpOFQk/zw/F3OUXvfgUry0poOTlTyosON0urlW9hGuf0NHH970s6AZ2bfkM8o4f+bwvers+bj70+/0+yTJPhbhEU8hzd/PDmUD3CEz2/HIF04Y2KIdCuhK9h45ZwJuOP8wXHHmfqmcubWqx1MvGPLXI6j1NDTnZ8+XIf864Ji9wiGffM1M38o/88BpaPdTwOUBoey4ofBNNjMQiVWUmHSJZY7G8i/HWP5yWYeKb/lnbYL2pkDKkqUqXaVK7vDV/Mb0CWrhRXBYtBK7nuZ8RlnegZH/pNY8H1TlSpMWIYHsI5D/yf46Cx84bAYGA6oH78yDpqEhJiM6DYbLoGBkHz0CzX+IZB9j+dcPJ8+fise/eCJ/z8iMEdINHz4ML3/tFG9bJMImbPnzEMRM8EWJ35k8S/jP4+bgk8fvCSCqgQNAuaLX/OVZhONr/q0N2VAZigZp6T1mXMvtsjhvHSHpCnCJtX3KjostfoZzyV/ApTFr+6GeDj581Cw88F9eaB7zfYgzjawVtfzZ4Cpa/jPGN2HVdWfi0DpnpbLexGv+de3CoMBY/nowg2+ovmdnkC3/Ue/wlTFtXAN/zW62Sp9XW/5h8t/aVQwRa5yjpr0px39UqozXkuOiLafOUJRnERWXortQQUs+E+p7QyZqTQNR/0XS+qxay18Y0EoVF4VyiWv8rCRGseKiUHbRnM/wuvzsekWpR45o8KJ99N/HYKE/5R1GEgbLmTgSceXZ+2NKax4nV7noTq3Ayd84fOsDURphXK6q+pizw0sGyuUd2n3LP6eRfaLtWby6okj+n739Jb/tKjR/10WXT/5i3/MSqTPLXi5pLM8QZGgtf744vcsTvpZs2IUXVu1ALmMha1t4ctlWlBwXFiF8AOWWv3Cv5Cu1NJb/YIFH+/QjwzdV+/383CUnzcVPPnRov88rY5DWCRmRmNCcwxVnzq/J0pL9wWBb/mPupyBKHcySVa37KVr+rDCaGOrJZB/Rqpa/NFFrz9hB7Zo+Qfb58ysb8PqGzupkHwp0FSuRgnTydbDm5EWtcxn1j2v2ZfdjQ0eftjQFk8nEweEr9yzBS2s6vKxoof8VfzYAAG/7i7KLUo9KcuLlHYag/ADRRPuIGArF5Iun7oOzD05fpiIJRvYZvjDkP4gIZJ8o2chVPssODck+bY3RdUlly1C0YDNCVqss+3QVKig7rtbiKFTUln9rQyY0SMnXYXHLPywnxfmz/vH2dmVUEyFC0TrBofvmZi/ix7YIT1KbNaEJnzpxLz4TWuHnPmRC90rfB1Xo7WChv3H+Iq48ez7+dPExtepSTTEapKvRCmZIDpbsOeY0fxGBwzd6s0WLPmMTr6SzSzF/WhumtzcESw0KUo2cHNLakEGXL3nkbALGmTL57+or+3H+6WSfCnf4toZlH43mL1v+cSiUHaXsYxPCB0sxjp9t6y06fCZ13pEz+TqnV549H1//8+teGwmW/6wJTVizo1d7H+qJNCV70xpkHz1mTg16VB8MVsVIg+rxuZPnIZ+x8P5Bimwzlj+S4/xZSeSK6+KEfSbjFxceEdSZF6xZObtTtHQzlsUfPDnaZ2tXwY/zTyn7+KGenuYfY/lLa+IyxFn+hbKjlH3EqpuqDOWuYgVdfiLWpOYgAetfDw9+yCGHr4KD/vCpd+C3Hz9qSMMi48hxIGn3w8XgNrLP8EVrQxaXnrZvzReG12FMW/4sdFEl++RUso9LOTkwYs/Fkb9FeOXQbMbyVjZA1PLf0lWMLe8go+xSdPaV0dYok79s+Xt/dWvYqlCsRNctYG2pZB8RLBFrUmuQeyDOOsRQTxXBT2lrwJS2hsj2wQDrTZwsMhpoc7jkGxgMPcY0+cc6fCXZp1hxQWlAYMyKjbP8bYtgXGMWhXIRWYuA2szhGyb/rV1FlCv6JC8ZXX1luNRzOosRPnLUkq2RfWhMAnux7CjLO4iyj678Q6efdcwkHyBMNqLlv+9urdhvWhtmT2zCho4+bX8GC6yfcdxoiNNgNGFMk78jJXmJCEXqWARbu7yEpnGN4ZLCtkBoOvLf3OnlA7DzqSz/kuOGEsbOOWS6tiz0dn8hl3GN2XCegaYWfUu1lr9C87csQfP3Lf+MRUKzBLZA98QWdd0dUQZryNp4sM712fuDuKX82O3tzyAQ166BwVBgTGr+/3ncHHz82DlgkZsq73rYYrWwaZdH/pNbPVmCRe6IMoFDZc2fBCWJbcIHFLm+vSf7hDX/H5x7CJ4QspFFMHmlrSGrHLgYGFnJx8QtEl4oO8okL9siPO2dlW+Y1h6WaNhAIFdPZRgJzsY4XjeRMgajCWOS/K84cz6+etb8YM3MBLkla1vY4JP/pJacv82XCYTjZEeqbVk8H6BUcflsQZZ9tncX4dJonaCmfHzI47jGrFKyCs4fJavFV50SGzderLgoOxTv3ncKvvkvBwb9IYQPbkz2mTk+vHTeZ07cC4A+iWywHFkDQRy9G/I3GE0Y/k9jHcHILMkizdiESx2T/FLCaVaSyliE5wN0FipcIpJlHybjyOSYt+PJv60xq4xUYlDJE0mLXLNon9mTmvH+w3bn2y1CeIYvuxfi4jjTxzXg0tP2xarrzoy0qSstPZxASPhv3DH9O8EAPlsDXHX2fBwys31oO2EwrDCmNf9PHDsH33pwKbfOdcgKRM+cmdzyj3mobYvghL0n456X1mPWhCYU/WQt2fJnMxA5vl2OOJJDLMc1ZpV1ghj6o7IUyt6KZaJM5V1LUO2QWf7vP2x3nLD3ZBw1Z0LsKmTjm7LY1l3S7h8OYJp8nDY/EPIf6mHvI8fMwUeGcf6BweBjTJP/J0/YC588Ya/E45gjNpexeH15JhXFPdQZi+B9h+6OBbPHY8b4Jvzj7W0Aopo/g1yATbSUm/M2Sr1h8m9rzGJHj55U+xPTXax4ln/WskKft0jU4dvakMERsycktnnbJ47G7c+v0foChhOMsmMwVjCmyT8J17zvAExpzeN3z60BAExuyUdq5MdFfjDynOFr40zW0VnrsuwjFhlrzmews7fM3xMCtOYzsbKPqFHfcP5hylLSMnqKDlzqDTzs2i44eg/8bekWLpOxGYguKU3GPru14qr37p/q2KFCIPsY9jcYGxiQ5k8IuYYQspgQ8jIh5GFCyHR/OyGE/JgQstzff5jwmQsJIW/5/y4c6AXUExccvQdO3X83LscwZy8QEGuS5S+C1d7f2au21mXyt6xgaUg5Vr81n4FlkdhoH5HHzjxoGj64YGZMbz2wSCLWl7e/eQauPmd/P9rHO4ZZ/iPBgVst6sX9ZkwxGG4Y6NP7HUrpQZTSQwD8BcDX/O2nA5jn/7sIwE8BgBAyAcCVAI4CcCSAKwkh9V2lowZgJCeuG8sDexQPNbOIbckpzJytHYIFLy7Gspsiu5UNIDL5j2MrjsVF+/SDcVihNuYHsS1vBmARRGr7xJ17pMFws8FYw4CeXkppp/C2GQEnngPgN9TDswDaCSHTAJwK4BFK6Q5K6U4AjwA4bSB9GAwwfX/auEa+jTk/VeF/LG9AtvxVZRbGC8swzhjfGNnP2pfXrG1v9D4XR8D9rUWTsQhOO2C3SFubOguYfdn9ePLNrQBGl+UfrOE7tP0wMBgsDPjpJYRcSwhZC+DDCCz/3QGsFQ5b52/TbVe1exEhZCEhZOHWrVsH2s0BgRU6E5Oa2Cin4goW525L0TuqRcfHNweRRnLSFBAMIM1SzP/UNm8WIjuJRfRXajj74OmhEg2AN4t4ZW0HAODh1zcnnnukol6ZuCbD12C4IfHpJYQ8SghZovh3DgBQSq+glM4EcBuAS2rVMUrpTZTSBZTSBZMn6zNSBwPbur2FysUlIBn7qwiWrZErW/5Z24qsUjU+tAB7VL9n1rttWfjAYTOw//Q2AMBUXyKK0/yrlX2yNsGLX30Pvn/uwdG2LBJZmH00kb9x9BqMNSQ+vZTSkymlByj+/Uk69DYAH/Bfrwcgehdn+Nt024c1WIx6SPbx2V9l0TGCV4VaytJPU8KqVWwAsQjwvXMPxjFzJwEI/ANxte+rzUglhGBCc05JhKptI6FcQ1oEVT1r1+YvL1yAU+YPzXqwBgZJGGi0zzzh7TkAlvqv7wPwH37Uz9EAdlFKNwJ4CMAphJDxvqP3FH/bsMa2rqjlT2Msfyb7qMiRkX97U+BQjYPNyd/7y2YhU3zZJ85irZbI4g5Xyfuj0VqOu6a4dRBUePd+U/Gfx+8JAGhrNFHVBsMLA/1FXkcI2QeAC2A1gE/52x8AcAaA5QB6AXwUACilOwgh1wB4wT/uakrpjgH2oe5gq3FNbVNo/kryV0f7AEBTzrvl45ty6OgtwyIE93zmnaG2Rdg8n8B7z2YhYuSRDtU6fGOzlUch0YdQxeVVcycW7DEeV5yxX2hRGwOD4YABkT+l9AOa7RTAxZp9NwO4eSDnHWzcedHR+NvSLaGCZcfv7fkhPnTkrMjxjTGWPyuNMGN8I1Zu68GkljwOnaWPdmWkyy1/fxYiO2RVqFr2iS1t4O1jSy2OVtR6jCOEcOvfwGA4wcxFU+CoPSfiqD0nhrbt3t6oLGIGBI5QlaTDEqT+7YiZOHm/qTg3IfGKRQyxQWDe1Ba8vrET09sD/8PMCY043i/T/KVT98F3HnpTe/44xB3O2po7pWVUkn+a2j7zprbg0Fnt+MqZ8werWwYGdYMh/zqAVfxUWf4sYmZySx5nHaQvrczALX9fQfrW+w/Ex4+dE7L8n770Xfz1xSfNxfMrd+DJZVurtmJjS1X4+yanmHGMRKSp6pnP2LjnM8cMTocMDOoMQ/51AA/PVETisOxY3WpXMgLN3/vblMvgoBnxpXmDhKVqZR89WEXS6e2NuOmCw7F259AvvVgPjHLPhoEBhyH/OoBxvsryH9eYxdauYuoKl7YQ6pkW7NCqnbQxh6/2pZ65U1pwyv676Q8coWCXPtr92gYGDIb86wAxMUvGzRcegceWbsb41OTvtVGNFc+LzlVJZHHnYPWI5k5pqa7REYbRGL5qYKDC6EnRHAb45PF7Yt/dWvl7leU/a2ITPlrFohosvr4a8u+37JPi8NmTmpIPGoHgmv/QdsPAYNBgyL+GuPyM/fDXzx3P39di2cL+WP6MwtJE+zz6heN5uGqaM8SVkxjJ4FE+hv0NxgiM7FMP+BlgtSh/YHMrPv1nqqlQOXdKK07cZzJuf35NrORx78XHoLeYvBjMSIcpwGYwVmDIv45Qaf7Vt8FCPauQfdjflLOFIJFMf8xoX/zblHQ2GGswsk8dwEo/1MTyl8o7pAE7Nm2SVzBGjV3mG7tXbjBWYci/jujPAuq6NvoT7ZP29P2NDhpVMPfAYIzBkH8dwFb5qo3lzxy+6T9T7WLkfHZRVc9GF07wazWdsPeUIe6JgcHgwGj+dURNLP9+hm16n61W8x+79H/4HuOx6roz+TrFBgajHcbyrwMYfQyV7MMiVtL6m4mRPDiMw9dgrMCQfx1RC0ua9Mcqr3K2YGSfACbD12CswJB/HVDtik9xsEj4b7rPVEdgLIvYEJ+BwdiBIf86ohZcGkg41cf5p4WJ9jEwGHsw5F8H1NJlaHGrPP1n2LFpZyCG/A0Mxh4M+dcBtIa6D7f8q3L4enBT9iPQ/A37GxiMFRjyH+5g2bpVVfX0jjWWf/+x1+Tmoe6CgUFdYeL86wDGubVwoPZnkRF2aNr5h9WPstGjGX/57LGYMb4x+UADgxEMQ/7DHNYAQj3Tyk9sVmGo38MBu48b6i4YGNQdRvapB3zOrQWZ9qfaJNPu01r+xuA3MBh7qAn5E0L+hxBCCSGT/PeEEPJjQshyQshiQshhwrEXEkLe8v9dWIvzD1fUglS55V9NqGeVug+bIJhBwMBg7GDAsg8hZCaAUwCsETafDmCe/+8oAD8FcBQhZAKAKwEsgEdNi8j/b+/eY+SqyzCOfx96FdvQlktpui1ttQbqrTYrKQGRANJSL6tJY5oYIWpsVEw0xtTWJgQ1/iGJoiZGglq5eAFFjQ3RYLE1/iMtt94QCqvUwKawaqVqTCrS1z/Ob3aHyey2szvn0jnPJ9n0nN+Z3fPsu7Nvz/nN2TPS9oj4x2RzVEl08WLPTu/ND6NnCaea48RI83f3N6uLbhz53wJs4pXHmQPAnZF5EJgjaQGwBtgREUdTw98BrO1ChkqZmd7qcGoX3sxlQlf70NnVPi27MrMamNSRv6QBYCgi9rUcNS4Enm1afy6NjTXe7mtvBDYCLF68eDIxC3fju1dw/lkzufqiyd8eePQ6/w4+Z+TI/9Q0zhB84G9WHydt/pIeAM5vs2kr8HmyKZ+ui4jbgNsA+vv7T6v77M45czqb1l7Yla/VaPoT+QvfU/0jr8bDfKmnWX2ctPlHxNXtxiW9EVgKNI76+4BHJV0MDAGLmh7el8aGgCtaxn9yJmZiAAAGd0lEQVQ3gdy10emtGtJndfQ5r5s/m/f39/HRty3rKJuZnb4mPCkdEQci4ryIWBIRS8imcFZFxPPAduC6dNXPauBYRBwB7geukTRX0lyys4b7J/9t9K7G0Xgnvb/TaZ8pZ4ib17+Z5fNnd5TNzE5fef2R16+AdcAg8B/gQwARcVTSl4CH0uO+GBFHc8rQEzqdwgH48KVL2fXkMGtePz+nVGZ2uuta809H/43lAG4Y43HbgG3d2m/v6/zKndeeN4s/bLkqpzxm1gv8F74V1+kUjpnZqXDzr7iR62+6+fZgZlZ7bv4V13jB94R7v5l1kZt/xY1e6unub2bd4+ZfcT7yN7M8uPmbmdWQm3/F+WofM8uDm3/Fjd6h0+3fzLrHzb/izpjQvX3MzMbn5l9xE7m9g5nZybj5V9xEbuxmZnYybv5V52kfM8uBm3/FzRh5S0i/0YqZdU9et3S2LvnY25dx/KWX+eAlF5Qdxcx6iJt/xZ05fSpb1l1Udgwz6zGe9jEzqyE3fzOzGnLzNzOrITd/M7MacvM3M6shN38zsxpy8zczqyE3fzOzGtLpcJ94SX8F/jKJL3EO8Lcuxekm5+qMc3Wmqrmgutl6LdcFEXFuuw2nRfOfLEkPR0R/2TlaOVdnnKszVc0F1c1Wp1ye9jEzqyE3fzOzGqpL87+t7ABjcK7OOFdnqpoLqputNrlqMedvZmavVJcjfzMza+Lmb2ZWQz3d/CWtlXRI0qCkzSVnOSzpgKS9kh5OY/Mk7ZD0dPp3bkFZtkkalnSwaaxtFmW+mWq4X9KqgnPdJGko1W2vpHVN27akXIckrckx1yJJuyT9UdLjkj6Vxkut2Ti5Sq2ZpJmS9kjal3J9IY0vlbQ77f8eSdPT+Iy0Ppi2Lyk41+2Snmmq18o0XthzP+1viqTHJN2X1vOtV0T05AcwBfgTsAyYDuwDVpSY5zBwTsvYzcDmtLwZ+EpBWS4HVgEHT5YFWAf8muyt5FcDuwvOdRPw2TaPXZF+pjOApelnPSWnXAuAVWl5NvBU2n+pNRsnV6k1S9/3rLQ8Ddid6vATYEMavxX4eFr+BHBrWt4A3JNTvcbKdTuwvs3jC3vup/19BvgRcF9az7VevXzkfzEwGBF/joj/AncDAyVnajUA3JGW7wDeW8ROI+L3wNFTzDIA3BmZB4E5khYUmGssA8DdEXE8Ip4BBsl+5nnkOhIRj6blfwFPAAspuWbj5BpLITVL3/e/0+q09BHAlcC9aby1Xo063gtcJUkF5hpLYc99SX3AO4HvpnWRc716ufkvBJ5tWn+O8X8x8hbAbyQ9ImljGpsfEUfS8vPA/HKijZulCnX8ZDrt3tY0NVZKrnSK/Rayo8bK1KwlF5RcszSFsRcYBnaQnWW8GBH/a7PvkVxp+zHg7CJyRUSjXl9O9bpF0ozWXG0yd9vXgU3AibR+NjnXq5ebf9VcFhGrgGuBGyRd3rwxsnO4Slx3W6UswLeB1wArgSPAV8sKImkW8DPg0xHxz+ZtZdasTa7SaxYRL0fESqCP7OziwqIztNOaS9IbgC1k+d4KzAM+V2QmSe8ChiPikSL328vNfwhY1LTel8ZKERFD6d9h4BdkvxAvNE4j07/DZeUbJ0updYyIF9Iv7AngO4xOUxSaS9I0sgb7w4j4eRouvWbtclWlZinLi8Au4BKyaZOpbfY9kittPwv4e0G51qbps4iI48D3Kb5elwLvkXSYbHr6SuAb5FyvXm7+DwHL0yvm08leGNleRhBJr5Y0u7EMXAMcTHmuTw+7HvhlGfmSsbJsB65LVz6sBo41TXXkrmWO9X1kdWvk2pCufFgKLAf25JRBwPeAJyLia02bSq3ZWLnKrpmkcyXNScuvAt5B9nrELmB9elhrvRp1XA/sTGdSReR6suk/cJHNqzfXK/efY0RsiYi+iFhC1qd2RsQHyLte3Xy1umofZK/WP0U237i1xBzLyK6y2Ac83shCNk/3W+Bp4AFgXkF5fkw2HfAS2VziR8bKQnalw7dSDQ8A/QXnuivtd3960i9oevzWlOsQcG2OuS4jm9LZD+xNH+vKrtk4uUqtGfAm4LG0/4PAjU2/B3vIXmj+KTAjjc9M64Np+7KCc+1M9ToI/IDRK4IKe+43ZbyC0at9cq2Xb+9gZlZDvTztY2ZmY3DzNzOrITd/M7MacvM3M6shN38zsxpy8zczqyE3fzOzGvo/oCXxo3Z1n/cAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mV5jj4dThz0Y"
      },
      "source": [
        "In addition, `avg_final_reward` represents average final rewards of episodes. To be specific, final rewards is the last reward received in one episode, indicating whether the craft lands successfully or not.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "txDZ5vlGWz5w",
        "outputId": "c7c5e9ca-6329-4ee2-f3d6-a1ba46b5aea2"
      },
      "source": [
        "plt.plot(avg_final_rewards)\n",
        "plt.title(\"Final Rewards\")\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAEICAYAAAC3Y/QeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO29e7wkVXUv/l3Vfc4MM+MwPIbnMDwHFVABJ6jRgA9UMA+MUX/k3sRcrwlqNNFcE6PxJj/NveZnkqsxRhN/JOIjapT4gkQTBUUQlccAIzA8h+cwDDADzHvmnNNd6/5RtavW3rV3dXV31TmnT6/v5zOf0121a9eq6qm1V33Xd69NzAyFQqFQjBeiuTZAoVAoFLMPdf4KhUIxhlDnr1AoFGMIdf4KhUIxhlDnr1AoFGMIdf4KhUIxhlDnrxhJENFuIjqhhn4+SERfrMOmuQIR/ZCIfnuu7VCMFtT5K+Y1iOhBItqXOnvz7yhmXsbM9zd87pcSUZyecxcR3U1Eb27ynArFbEGdv2IU8Mupszf/Hp3Fcz/KzMsALAfwBwD+kYieOYvnz0AJ9JlV1AL9j6QYSRARE9FJ6efPEdGniOjbaYR+PRGdKNr+LRFtIqKdRHQTEf1Cv+fjBN8B8BSA56b9RkT0PiK6j4ieJKJLiejgdN/nieg96eejU3vfkX4/kYieSo8/iIj+nYi2EtHT6edVwvYfEtGHiejHAPYCOIGIXklEdxHRDiL6JAAS7U8ioqvTfduI6KsD3F7FGECdv2Kh4EIAHwJwEICNAD4s9t0I4HQABwP4MoB/JaLF/XSeOupfAXBo2j8A/B6A1wI4B8BRAJ4G8Kl039UAXpp+PgfA/QDOFt9/xMwxkmfwswCOBbAawD4An3RO/5sALgLwDAA7AHwDwP9MbbkPwItF2/8F4HvpfVgF4O/6uU7F+ECdv2IU8C0i2p7++1agzTeZ+QZm7gD4EhJnDwBg5i8y85PM3GHmjwJYBKAqdXMUEW1H4pS/CeB/MPMt6b63AfgAMz/CzFMAPgjg9UTURuL8X5LSNGcD+CvkTvqcdD9Su77OzHuZeReSQescx4bPMfOG9NrOB7CBmb/GzDMAPg7gMdF2BslAchQz72fmaytep2LMoM5fMQp4LTOvSP+9NtBGOsC9AJaZL0T0h0R0Z0qFbAdwIJKouQoeZeYVSDj/TwB4udh3LIBvmoEJwJ0AugAOZ+b7AOxBMgj9AoB/B/Bomi/InD8RLSGi/5+IHiKinQCuAbCCiFriPJvE56Pkd04qM8r970VCA91ARBuI6L9XvE7FmEGdv2JBI+X33wvgjQAOSh35DgievArSyP6PATyHiMwAtAnA+WJgWsHMi5l5c7r/agCvBzCZbrsawG8hoWTWp23eg+Qt5AXMvBw5NSTtk6V3twA4Rlwfye/M/Bgz/w4zHwXgrQD+3uRGFAoJdf6KhY5nAOgA2AqgTUR/hiSK7xvMPA3gowD+LN30aQAfJqJjAYCIVhLRBeKQqwG8E0k0DwA/TL9fy8xdYd8+ANvTZPH/28OMbwM4lYhel9JLvw/gCLOTiN4gEsZPIxk44n6vVbHwoc5fsdDxXQD/CeAeAA8B2A+bJukXlwBYTUS/DOBvAVwO4HtEtAvAdQBeINpejcS5G+d/LYAl4juQcPYHANiWHv+fZSdn5m0A3gDgIwCeBLAGwI9Fk58DcD0R7U5te1fT8yEUownSxVwUCoVi/KCRv0KhUIwh1PkrFArFGEKdv0KhUIwh1PkrFArFGKI91wZUwaGHHsrHHXfcXJuhUCgUI4WbbrppGzOv9O0bCed/3HHHYd26dXNthkKhUIwUiOih0D6lfRQKhWIMoc5foVAoxhDq/BUKhWIMoc5foVAoxhDq/BUKhWIM0ajzJ6JjiOgqIrojrS3+rnT7wUR0BRHdm/49qEk7FAqFQmGj6ci/A+A9zHwKgBcCeAcRnQLgfQC+z8xrAHw//a5QKBSKWUKjzp+ZtzDzzennXUhWOjoawAUAPp82+zySdVAVCoUHl63fjF37ZwrbuzHj0hs3odMdvFz/tt1T+M/btwxj3rzF5u37cNVdT/Rsd//W3fjJfduy7zc99BTu3LLT2/bux3Zh+97p7Pu3btmMPVOd4Y2dA8wa509ExwE4A8D1SJa5M//jHgNwuKf9RUS0jojWbd26dbbMVCjmFe7cshPv+sp6vPdrtxb2ffmGh/Her9+Kz/80OI+nJ9782Rvxti/ejB37ioPLqOP8j1+DN3/uxp7tXv7Rq/Ff/vH67Puv/cNPcf7f/sjb9jc+cz3+6UcPAABuefhpvPur6/Gn37q9HoNnGbPi/IloGYCvA3g3M1tDaroGaWFRAWa+mJnXMvPalSu9s5MVigWPfTPJgl9bduwv7Nu+J4lAn94zXdhXFZue3gsgeYtYaNi5v/6IfO9UB3umk373TCW/zeO7ir/NKKBx509EE0gc/5eY+Rvp5seJ6Mh0/5EAer+bKRRjjDLXzKV7y9HXQsYKxAyY9a+Gue/zAU2rfQjAZwDcycwfE7suR7KQNdK/lzVph0IxqihzzqSee9bBYLirH9KIDqFNF3Z7MYDfBHAbEa1Pt/0JkvVHLyWityBZV/WNDduhUIw2SpZb1ZVYZw8xJ/+A0b/vjTp/Zr4W4eDlFU2eW6FYCKCS8L5sn6IhcJHuGdWfQWf4KhQjgHLOv4b+Rz2MnSXEzHnkP7emDA11/grFPMaIBpULFozRp3sM1PkrFCOAphyOUkf9IeZiwndUoc5foZjHqOKbF4gvagx1OmuWUs8Rv/Hq/BWKEYBPU15n0D7abqwcdflo4+xjV+o5om9P6vwVihFHHZONRjyILUVdl8ZOonfUb5k6f4ViHqNsAlGdk4tGfbZqGeqiZ+JQ5F9L77MPdf4KxQig1H8N4duM41rIkX9dZYvY/TDi90ydv0Ixj1FGJ9fK+Y+4IytDXW81wch/REN/df4KxQigzDnXMslr1MPYEtSX8E3/mu8jfs/U+SsUI4o6A86FHPnXBXOPXBppRAN/df4KxSiglPKvwXMvZN9fW+Sf3iVzv0d9wFTnr1DMYzTN+Zs+4gW4mIuBy9EP3k/y1+1Odf4KhaIxlEX3ox6BNo36dP5pxA+N/BUKRcMoczC16vxH3JGVoT6df/o3trePZtyvzl+hGHmo2qcctV2Zs3zjqN8xdf4KxTxGaeSvOv9KqOvacp2/vX1EKX91/grFfEYVfrkO57aAfX9tF5fp+7Wqp0KhaBrN+5ckbK1LETMfUZ/aJ1TXYTRDf3X+CsU8RpXZpFrVsxx1V/XUZRwVCkXjKKMW6tWXj7orC6Mueiaf3KW1fRQKRcNweWZvmzo4/4Xr+ysPa70GCbM3NNlr1KDOX6GYxyjX+dd4nhr7mm+o6qR7tYuzSV42RjTwV+evUMxvNKspz8o7jHoYW4KqtE+vVkWVz2jfszlz/kR0HhHdTUQbieh9c2WHQjGfoTr/4VEX7ZNF/qrzHxxE1ALwKQDnAzgFwK8T0SlzYYtCMZ+Rc/7NeucF7fwrXluv2na52mdh3Ky5ivzPArCRme9n5mkAXwFwwRzZolDMW1TxM/WUdF4YDs2HqtfWq11O+9h/68JXbngYT+zcb237wk8fxMXX3FfviVLMlfM/GsAm8f2RdFsGIrqIiNYR0bqtW7fOqnEKxXwBB5KMgC7mUhX9JHxLq6cisIxjDb/EYzv2433fuA2//YV11vYr7ngc37ntsaH792HeJnyZ+WJmXsvMa1euXDnX5igUc4IqfmsB++1a0M/9KRso3Mlddd73mW5SKvTJ3dOFfVFDOYW5cv6bARwjvq9KtykUCoEyYUkdk7xMDwuFx/ah6kI1zOX3wZ3kZdo2mfCNmRtbLGaunP+NANYQ0fFENAngQgCXz5EtCsW8RRW+Wid51QPucbfdyV113jPTl+vnmZubR9BuqN9SMHOHiN4J4LsAWgAuYeYNc2GLQjGvUVJHplapZ31dzTv0o/YpfwOyI/46I38z7Hidf0Pef06cPwAw83cAfGeuzq9QjAKqcf51FHZbuO6/stqHGcxhT9tkQbcs8nfifAbXumKbxLxN+CoUivLa8VreoRoqq316tHWremaRf42/hC/yb4r3UeevUMxjNM35G2ezkCP/qsls5oqlswMzfYdBqCvGwlP7KBSKCuAyqqFGMngB+/7qbzVcPsvXLNweO28AdUTmnL1FFLc3RfvMGeevUCh6Y7Z0/gvY9/dB+3A57ZMV2bMln3UgG0ecAX1BJnwVCkVvcBnFUKfzWcjev+LQFjNAVTj/2P5eh28O9cVozvkr7aNQzGP0Fh7WpfNfuN6/enkHLr3hLgVXbz0kv/dvkvZR569QzGdkDqfoaGYj4bgQUPXaGOXJ4byks9H5J9vrnIGrkb9CoQDQQ31Sw6IiJqpc0OUd+lL7lOwX7eTfOhDqK+a612rOoZy/QjGPUeZoanXXC9f395XwLVX7ODN7c53/8Ai+RTAvrPIOCoWiGqpMOqqF8x++i3mLvko6V+hn+PetIkIDSZO0jzp/hWIeoyypW2u6cQF7/+rlHarRbC73X4dzDtUJarKwm3L+CsU8RuniIjXONF3QK3n1o/Mv3W9/aKSqp6+2j3L+CsX4oWmXbPxKxZL3I4l+aJ+45I6bdQFMi3o5f/9547i58g7q/BWKeYxKnL9W9SxF9TV8UTrauk6/zluWJ3x959TIX6EYQ5TQPjW+Fyxc19/fJK+ypq7Tz3n64Z1zKPJnZk34KhTjiLKSzrXqzRew9688yatXQ8fp1wkuGUhU6qlQjCEqJSBrOc/C9f5VKa1ezdxlHBvR+Xts0shfoRhj+PxSHQGo8SsLmPLvo7xDeUu3mmetnH8g485gRKr2USjGD+UzfG31ySAwNMPCVvvUHPnD/ltH6B+6/7FG/grFeMKtIW/tq1VnvnC9f3/LOFaf5FXnMo7hBdy1qqdCMZao4ri0vEM5qid8e0zyYv/fOpBN8vJJPTXyVyjGD6XlHXQxl0qoem1xD6G/GRryxG8D5R0KBf21vINCMZYopyHSv7qQYymqD5LlyziaFbyQJX6HscrpuyTybyrhq85foSjBz/9/38df/uddc22GX+1TY/8LOeFb9dqYy9vmM3ztv42Wd2hwkldjzp+I/pqI7iKiW4nom0S0Qux7PxFtJKK7iejVTdmgUAyLR3fsxz/88L45O3+V8g51jAILmvbpo7xDWVu3mmetM6xDJZ1HlPa5AsBpzPxcAPcAeD8AENEpAC4EcCqA8wD8PRG1GrRDoRhZcAnFUG95hwXt/as1Y0nt+PcDxci/DmTndcL8Jqt6Nub8mfl7zNxJv14HYFX6+QIAX2HmKWZ+AMBGAGc1ZYdCYcDM+PTV9+HJ3VONn+uJXfvx+Z88OFQf6zdtx7dv3VLYvvGJXbj0xk2ZM7rq7ifwx1+7FRse3VG572vu2Ypr7tmafQ9F/v95+2NY9+BTfdltcO292/DxK+8Z6NgQ9s908aF/24BPfP9efP2mR/Cje7fiH6+5vzw34nzfsmMfLrn2AU87u6jzJ75/L2a6+WjATuRvbtptm3fgsvWbB7wiYMOjO/CNWx4BMLuR/2ypff47gK+mn49GMhgYPJJus0BEFwG4CABWr17dtH2KMcCWHfvxkf+4CwctmcD/83PN/p/63S/ejHUPPY2zT16J4w9dOlAfr/3Uj8W33Cl97abNuOTHD+Bt55wIAHh67wy+um4Tlh/QxqlHHVip7zddcgMA4OgVBzi92/jo9+7GmsOXYe1xB/dt/2985noAwLvPPbnvY0PY8OhOfPbHDwIAJloEIsJ0J8YFpx+Fw5Yv9h7jjgtv+dw63LFlJ8477QgclV4/kETfsu3HrrgHL1lzKM5cfVDSj9OfifzvemwXPvztO3HB6QU3Vgn/csPD+M5tjwHwL+YyL6WeRHQlgCM8uz7AzJelbT4AoAPgS/30zcwXA7gYANauXbuA30kVs4VubEv1msRTe6fTc5bwCAMiZk6iT8erdYa4sFDk3GUupUJmG11xjcn1svjsh0tp7dg3U+jLtHNvg7wvGefv6dftqx/0OnZelndg5nPL9hPRfwPwSwBewfld3AzgGNFsVbpNoZgVzG5ys54HV9rMzN71ZkP1Ycpg/ErwnvD8ygdIRyltLnOg7q7QNSdqHy5scz/nM3zF+Yf4TyVtd/+3xNzcAu5Nqn3OA/BeAL/CzHvFrssBXEhEi4joeABrANzQlB0KhYE7Lb9RNHiKmE30b28fxgGFHHzMPPSbUp2T0cokkVXPnzl/X8mMQr/Fc/hm+A4X+UvbnIQvj2Ztn08CWATgivSCrmPmtzHzBiK6FMAdSOigdzBzt0E7FAoAclJU1fbDO626HlxpiYn6XefVHYKeCV0ql+yripkuY7Jdz40IOdky5+vuMbNoq0X+xX5dyScw2FuX219im2MTmqvt05jzZ+aTSvZ9GMCHmzq3QuGD76EtwzBOr8l3CxP1u/YN44BC15rlF4bATDfGZLsekiH0dlP6NheifTx9FO6pL/L3dFsb7eNJ+I7cJC+FYr7BVWv0wjAPtEFdz60/ArW/D0f7hM47/EA2M8wriYPQAFd2CvcNKV/DwE34+rYVcwxZdU9hyzBJ8a4V+bs6/3qWifRBnb9ibOCW5O2FYXjcJkskh95ghkn4hteQHT5HMl2j8x+E9nEds3Gm7hG+qp7y0t2VvOqK/Mt+tybX8FXnrxgb9FuKt47EcF1Rm8v5A0WHN9SbSojz91Ah/WKmO8cJX+d7WeTvOmJXZSXPJc9Zm9TTR/sM3HM51Pkrxgb5tPxZiPwHPrI3fFJDYFh7Q051+EFwplNn5B/aXl3tY7yp25dXPiuOdSP+unIu5Qlf5fwViqHRr159PlW6tCLQ9K/rlAdx0r3ehurQ+NfJ+YfebsreekKRf3HA4KLax/ps6Lb0u9N20Dev8oSvruSlUAwNw/1WdZJ1qGea4P5DuYtBIn9zTOjIOiL/Ojn/0G9Szpvb3/N1i93BE4UbIdvIU/jyA4O+eUlWzJ/wHajbnlDnrxgblFXI9KEOtU9drp899EOB8x/Ax5qovCzhO584/0ESvu6vEIr8k+R2+FB3tm8db15A74FLF3NRKIZEPj2/WvuhIv9soKk/8g9x/oM4n+mUjw/SPlykQvpFZ45pn1B5B/eYJJoPO3T3s3vKgSP/Etqnydno6vwVY4M8YVcx4VtH5F/Ts2txz2YQK0T+/Z/MROWhIxnDX0OdtE9oMK1Sh9/AUCsFZQ+KA4Uv1xJqO6jW39L5ezK+SvsoFEPCrc3Su/3g5+r3LaMfhFRLg0SJWTK2REI5qPNvRZSeo07ax7+9XOrp0D6Z2qdI+xTmTlh0m/sWUE/CVw5CXs5fE74KxXAwj1hVKmYY2ic/Z/2hv+mzDtqn0yPhm8gfB7uGVupla5V6DqL2Cewq0D6eK/W9cZnPBannoGqfkshfJ3kpFDWg38h/uBm+6TmbqIUfiPzrsNfFMJF/lHqX2SjvUJo0db5nah+fzr+QBxDncJLuhTevQXX+PWyP1PkrFMOhXyqmHrVPPZG/7CVUmnqYgSYYtXqcXFWYyH+uyzsUSjqbYzyO3r1Un8oKgHfhl4F1/qXJ6hFcw1ehmG/I1mCt6JBroX0a4PxNl3WWdyiN/AfsM2qA8x+snr/9Patn5FnJq5DwDXxO5j/YbQd984p71fMfqNfeUOevGBuYR7Oy1HOohG9/FFPV/oBigTGDJspR+FQtVZEnfGcj8g8f00TC1ysLHfAye5V3aMr7q/NXjA3ikNcMYBhnmp2zwRm+rn1Vz+VLeAcllEPM8jKTk+a8vIMb+afetJjw7SH1tPh/z+A7KO1TovMHq9pHoRga/Uf+w0fSdbl+n+pk0IRvP2Oad9ZrRRjnP12j2qee8g7+Y9gj3wxN8oLnLWHw8g5lkT9rwlehGBb5SkyzF/k3UtsnIPWsaq+vXWl5h0Glnql36dRwHw0GquoZ6suT8C0z1U261yG1BcoHrph1kpdCMTyyiLla8zoSqI1M8srmZQ1G+/jahcU+PDCX3U61nnOt83evl7LtdjufgicU+RffEeqJ/F1oVU+Fogb0SfnXFLXX4/1dmSFQdBrDRP7BhC8PfwVzrfN3LyDX+fsi/+I23+ck8q+H9pGDq9sDQyN/hWJo9FtsbRifFaJm6kBe3sG/vRd8kWb5JK8BnVp63HSd5R0GSfj2o/ZxjnXLOMu2ZeWf+4G0ozjPQKWeCsXQMA9W1Ue0Hs5/6C6SflB0EIMWdvNFySFenzH4NRh75jryL6p9EvjUPsUZvn7H7Kt2Wgft4+1BJ3kpFMMhmxlbWRUzvOduUuo5qPPx0j4hzt9DhVSFOW52dP7VE75h2qfI+bvyTtlnWX6gH8RW5F/8rGofhWJIsPO3F+ZV5O84HmBwnb+f9glvG/QSmoj8w7RP+JhQwtfty7dqmfXGJT77Of+wDWWwIn/PYDOyCV8ieg8RMREdmn4nIvoEEW0koluJ6MymbVAogHDEHMJQOv8suVx/5B8qUFed9ilu85kZKh1dFcae6U5996BJnT+8ah//Z19+oI7FXGx6jy1760ajzp+IjgHwKgAPi83nA1iT/rsIwD80aYNCYZA75Grt4xoevkHdXoF7tvYlf4uR6xCRf5kNA16E8WlzPsPX+Z7N8O1T7SO/+BLh9dA+4nSZvc2g6cj/bwC8F/b9vwDAFzjBdQBWENGRDduhUAipZ1VuPPk7yMNnzjA4X162r/gG04pouIQvA5u378PHvne3eEOyz3f75h24+eGn8aF/24B9092e5xmW9rl98w586fqHsH+mi098/15Md+KBJnmFMr5ddjh2FAeK793xGL5/5+MAPJF/n29eV939BL51y+ai7Q7t8+1bt+B7Gx7L+m8q8m830y1ARBcA2MzMP3Mq1R0NYJP4/ki6bYtz/EVI3gywevXqpsxUjBH65bDNwzzMAtrfvHkzli+ewPOOWdHXcQVH4qNlRJuJVnXn79f5M9755Ztxy8Pbcf5zjsSzj1yeS2PTNr/0d9dm7V95yuH4+RMPLT9PlvAdbAA059u2axp/c+U9eMbidj31/MUxro7fDQx+ePdW/PDurXjwI79YUFwVOP8eA/2bP3sjAOC1Zxzt2C5tZbzjyzcDAO7+3+cl9jbk/Ydy/kR0JYAjPLs+AOBPkFA+A4GZLwZwMQCsXbu2fuJUMXbol8MehvYxp/jGLZsx0Yr6dv5lNvoWcJ9oRUPp/GPOI/SOWde35H51Kjj03CEP9/hOdZK3jD1TnXoKu4kF3OWuXgvX2Gqfojh20BLg0nb37ULaWzeGcv7MfK5vOxE9B8DxAEzUvwrAzUR0FoDNAI4RzVel2xSKhuFPlIaQOf8hWddBqJ8CH+3R+cs2k60IeytQMUDASTGjlZZj6KShaFmOpAqV0/UMUoOgnWodOzEPFPkX1T455+/SPlXZozrr+XftUaVwvpFS+zDzbcx8GDMfx8zHIaF2zmTmxwBcDuBNqernhQB2MPOWsv4UijoQmhkbQvZQDvTs+aO5qvDJEPPPRc5/ohVVrkUUSvgaJ2uuu2zZy17OX2rmh53rYAalblyUVxoMEvnHsR29+2r0w9lvta0h4etbUMb9PC8j/wHxHQCvAbARwF4Ab54DGxRjiPzZ7I/2GXaSzSByT3+5YfM5bSOdf5sq0w6hSV4tEWEDYl6Ex/5eJRvkOYaO/Fu5XaHTli/mYsOifZyd1at6+hK+4WND8FUWdT83pfaZFeefRv/mMwN4x2ycV6GQyGf4VmtvHuZBEr42RTA87ePjgi3OP6oe+Xt1/uBszV038vc5xF6VOrtOlDwMIjEjN0j7lEb+YaqoQKeV5VqscxffEgapAluk94qfR1Lnr1DMJ+Tpx6pO0nD+w6EO2geQaqUi7dNukVetUr1vO8I22+T5JHrRPnKAGZb2Mc6vE/NA5R1cGNvcyN+3hq+ETREVB9FBEr7uvfEtGzmM2qwM6vwVY4N+19XN1T4DRP6efvpB2SzcLPK3pJ45L96z74Dap5Vx/ibhWxL593D+loJlyDleWTmGmIPRdRV1lPvdvYQ4Lh887dr+xUFxkISvb6KZPEeTUOevGBv46JIyGEczbNw1SOBbNgs3JPV0t4UQquqZJ3zTbc5gI9Ef519X5B/XUt5B5kzsyL88G+S+JRTzBcMP8uz50pTOX52/YmzQ7zKOGe0zkM5/OOfnc2a5/QlcqWfVc3kjVE/kn/c1CO3jcOlDQEoz6yjvkEf+Luffg/Zx3mbqcP4Fuy3paT3BRwjq/BVjg7JI1odu5vznXucPFN9cpDOaaPvr1Xj7Dko9jc7fHmSGTfjWFvl3E86/5ZFflap9Ak66687wRTntY3H+voTvIGqfsoRvFvn3328VqPNXjA1y3Xo1Z2SYjUGknjbn3//x/lm49ugl22Scf4Vr8+cTcqdqZu+W3a9+Iv+61jQwOn+f8y8v7+By/rldLs0SMjV25hgweyZ5DfKGVyb1TP9qwlehGBJlkay3/RAJX18//aB8xqr9FxCc/4CRf8z5JK/MsZfkSHpy/k5ydBgYB2nUPhO+yL9U6ul+l5G/TbOEKMGZOO5ZB2gQtU/Zmgx1VJUtgzp/xdhg0MJuA1X1FCepO/LPpJ4ezr+S2qfHJK+ZLPI3+/qP/O11aYfz/saebsyIY/Qf+RfUPuavW9sn/FvNdO18QOzJD5SuJhbKVZSofTLaJ9jrcFDnrxgbSJVHFXRrivxr5/zjYr8TqUa/CvUQquppdP7GsecL3hf76F3eIf88bORvaKhOHKPLnL3lSPQT+Qc5/xLaJ8lxyLeEcCLZe3zgTam4cpj83Czpr85fMTbot8hkfWqf/o/38fJlap+c9undd2iSV8uhfbLI39NHP5H/sJy/KTTXjcsSvmWcvw3TtGv783SSV4D26caFiWtuNF9mQ+h+FSP/Iumvkb9CMSR8M2PLkJd3GPK8dUgAIWYoZ4nYfN9Eu5+Er7+NUftMd91JXsX2vZZmrJPzn8ki/8Q5+yL/8vIO7vf8unzVUn2Y7saFtm77snsvnX9VGbDZo5y/QjEkcg67WvtB1BsGNpdcE+3jTL6S9tKwVxoAACAASURBVE0OmfBl5kxVMtOxB5dBaJ+4Rs6/0x028nc5/wDtg/D9czn/ZE6A02+JDdPifklbC7OMHUURoGofhWJ4lESy/uZhzrsfDFLewGej++bi5fyrOH/vBLI8wsw4/5IcSV/lHYamfdLIP9X5m9yEdb4Sc9zLzWifgtQzPP1vphsXBopC5F9ig+T8y+6NnStJacdwt0NBnb9ibFDGYftgnORArsvzEPeDkIOWXctBpS+df9pmQjhRyXfnnH/4+nstzWjsb0c0mNpJHGTs6abRdtun9im77oLaJx08Halnovbx9zPdiZ0ZvsXIv5T2EZPi5O+mVT0VillAHslX5Pz7bB8+b//H+OWYZZF/H7RP6nwMx29sNN0Zx+7mGCSqVvVst2igwU/23xFSz27Mlt0GZYlT9+wZbRZ7FnMJmDrTjb2OWaLs3svrKYv8fVU9R2olL4ViPqJfzt88zIM476E5f88xrlrJ4vwHSPi2rcgf4ch/AM7f2DERRQPdP9n/TDqSJAlfeGmf0pmyBdonHzyLVE4Z52877b4if0n7xP7PQKAmkkb+CsVwyOiSPtU+w3LWtSV8Hc7fUvv0w/lntI8d+bvOfyjOXwwwg7w5SWeZR/5xGvn3TvjaUbo/ui4WdgtTfC7n7ysFUTXyLyt94aPTNOGrUAyJfmf4lnHeVc+V9NP/8d6Er8P5S/Sl8xd8fN53rmZxpZ6+8/Uq72Dsb7eiga7fon1M5N8djPYpJHyzxVzg6PxLOP9u7ET+xbZVdf7yDcEdQ33zIzThq1AMiTySrda+jPbo77yDRP6+fpK/Pgc1WMLX5fxN5O9y/sU+elb1jA3tMxjnPy36t8o7sF/tUzpT1jl9ljvph/PvuJx/URlUdu+nA5G/O2B0PLSPJnwViiHRd1XP2KZZ+oHN+fd9eEDtEx6M+qntIykZAwZnEbFx7L7ko0FVnX8rXV6yX9gJ31znH6rqWVojJ1DV01feoay2j/s20R/tIxx+ScK3K17dMspfnb9CMRzKIlkf8hLKw513oMVcfLSP+evpztTz72cxF5v2CXP+Sb92H/0kfAdT+wjO3+j8Y0Y3RqC2j/2dXT5HIOP82eH8y6p6ujp/9pR3qCj1LEv42pG/oX2U81cohkK/K3kNFfmXOM5+zi2Rq49KaJ8qVT19tI+wc7pbJfIvP498uxhW6mk+d1LevUpVT1fFY7U19Fkfhd1czt/3llA+ySug8y/JG2jkr1DUhIwzrzjjtt9JYda5EHac1c4dPsbn3wfS+bfsyJ/7iPyne9E+JuEbDZbw9ZVDyOr5e2f4hk9SvP8y8re3hqWe7gzf4ltC5fIOkvapEvmr2kehGA7cZ+QfDxH5W/3UJPUse3OZHCDha0/ykjN8i7mFoiSxV8I3+TsxqNRT0CTGISb1/Bktj9qnvEyC2zb968zwLeX8O7G1L46LQUQdOn97HYTk70iqfYjo94joLiLaQER/Jba/n4g2EtHdRPTqJm1QKAz6VfvkM3wHP1c/57PO7Z3hG+4vX3x9QM5f9OvW8wfsiBSorvYZXOopzt3N30i6XFXnX3SiBhbn7wxwQbWPowv1Naus8y9N+M4e7dNupluAiF4G4AIAz2PmKSI6LN1+CoALAZwK4CgAVxLRyczcbcoWhQIozpDthay2z3CB/9A5g2xbyT7j/Kucy7RpFXT+yXYjs5Rduc61F+efv13Ux/mbWbZezr/kHCF6pmzNBBfTKe1DlCfH+5vhW62qp2V3Fvk34/0bc/4A3g7gI8w8BQDM/ES6/QIAX0m3P0BEGwGcBeCnDdqiUOQrUwW8/4ZHd+A/bnsM73nVySCiUprlR/duxRevewjveNlJeOTpfYgIOO+0I/GTjduwe6pjn1cc/uTuKXz0intw6lHLcfCSSZz/nCO9tvgcySXXPoBHnt6LbbunCvvaWeSfOO8/+trP8NSeabzg+IPxzpevsc7/8SvvTY4R3Pm31j+Kw5cvApA4qi079uFPL7s9t0c4LCLjDLnAR199z1Y8sHU3DlmW9DXRiqyI+EP/tgFbtu/HX7zuOVgy2cKf//sdeM8rT8YXr3sYt23ejlc8+3Dsmergsz9+MDvGvHXMdGMQIVDYDXh853588gcb8We/fIp1zz/74wfx+uevwqlHHQgg/z3u2LITP7p3W9bur797N17xrMMKfQPJ20fMjBYROsz47obHsGXHfquNuUf/um4TGMAtD2/H6ccciEXtljVv4aEn9+DSdZvwp794SvkKZDCcf7DJUGjS+Z8M4BeI6MMA9gP4Q2a+EcDRAK4T7R5Jt1kgoosAXAQAq1evbtBMxbgg43oDz9sVdzyOT161Eb//ijWYbFMh8l+/aTsOXjKJ1YcswQcv34D7tu7B6ccchO/f+TiiiHDeaUfiH390P7btnnbOm5/wL75zF75+8yMAgHNOXhl0/h1PSPjP1z1U2HbARAuvf/4qLF2UPMrdmPHI03tx2fpHQQTc+/huy/nftnkHAOC5qw4slA14fOdU1sePNz6JWx/Zke2Tzv8VzzoMV975hLe88jdvfgQ3Pvg03nveMwGY8g55H8ap/8YLj8WKJRP48vUP4+w1K/E3V94DAPjBXU+AiKzzmXvRiRkR+Wv7dGPGT+7bhn++7iH8txcfh8OXL7b2v/HTP8WGPz8PQPJ7HLpsEtt2T+M/bt9itfv+XU/Ah06ccP6tiNCJGZetfxQA8IzFbeza38n6BYAv/PQh3LFlJ7ox46s3AmeuPgivPOXwrK+3f+lmMAO/duaqbGD89bNW47L1m7F3OidAzH0bdjGhEIbi/InoSiK63fPvAiQDy8EAXgjgjwBcSn2krZn5YmZey8xrV65cOYyZCkWCjMP3e3/zIMpyAkD+UL/2Uz/G2X99FQBbBjrTjS0H1XFmjkrnH+J+XUz14NQNTj9mBf7Xa0/LqBBJ3yybbBciS7Pvzy84Lagi6abXZG1Lr/dPXvMsnLH6oKydC0PN5HmFXOfvljjwSWljzs/15hcfl/Up94fKO2S1mOKiBt9K1jLwujNX4bBnLKqUIzH9x8zWZLo1hy3Dq089Iu83ztvm15Zeq5NYBpKaPabd2845ARecbsfA+X2Zh7QPM58b2kdEbwfwDU5+hRuIKAZwKIDNAI4RTVel2xSKRmEepdDzbh7QmQ4Dk8CMifxL+oxjxnSX0c7eKriQ+AvpustrwVRzSsZ/Zwlfzp3gRDsq2GL2tYiCEWUcF9U8xu5IHOfjzKe7seX8Jlp5PX+3xIHpM3QfjKPtOCcK1fM311pYpAV2fiNmBlEexVeBGawm2hEwlfdJThvTv2VbXPw/YY6X99UdizPOfz5G/j3wLQAvAwAiOhnAJIBtAC4HcCERLSKi4wGsAXBDg3YoFAB6F2ozvsk4KRPNl+UrTZScLThinJo4xl0EJD9fuOPpipG/cQyGwnEdb2gSURSFq0V2Yy6cP1/MnmDmhvkj/9hSzbREwted5dpLSmvmLnScgbAVoH3kYOJ2KccLZjOIUeXIP/bMMXDvXzb4uAOuGJCt64hEGYyomNZturBbk5z/JQAuIaLbAUwD+K30LWADEV0K4A4AHQDvUKWPYjZgHEKQ9nHoCel0QjRCHHPq8FK1TVyU/MmvvqqNPvTS0RsYlyHVPnIGr+SQ5TlbUTjyTwY02zYTIUcEMIWVRTMm8hc2mGZufZts8lbgLcc4f/deTAR0/hnVEqMwwruRf9Rv5B8nx8lZ0a2IrNNkg09hwPUPlET54NyKipF/Vwy4TaAx58/M0wB+I7DvwwA+3NS5FQof4sz5+/ebhy2TFsaSn/e37TInE4BagnZhe/ZnSNddGvlXdf6G9ski/7zfyZaP9uGsfcipmAHNd1xEBKa8nYuZDltvH1Lq6da0N45vf8cf+5l6Ra6DDhV2C9XscY9JnD8ltEvF6d5mcJkUzj9y7MgHn2q0j9xu3kTccyb7KpnYN5qM/BWKeQV3MRQXrvMvj/zNKz4szt84Ptk8FPmX0fpVaR8DEwxLpzrRioIJ36hn5B9y/si8kW/wSmrgCJ1/K0/4uiUOjN/dP+O/1skA7RMq75ANyF7aRzr/JJqOKPzW4e2f7XpILfLPgi5E/k7C19peQvsY20ZR6qlQzCvktI9/fz7JqRiputGnq/ZhFrSLo/aRA4f0NVVrwZTBRO8y4Wv6nWhTsQSBiPxLOf+A8yeR8A1x/qGErzvRKYv8ZwKRv6F9nIvwl3ewfxPXMqmGApDRPpXVPul9NW8jsk95TUAxER6K/M2AYuxx38TM/RnFSV4KxbyCuwC6i0LkLx7YKSc6Nbu6DudvnFqI6qk94Zv+bXkTvsXI34o0g2ofThRPnuOsCNpjoinBIKWeQHLvZZ9SnTPVw/m7P1co8reWZ3QOirI8Rf49Iqo8yJqE76IJQfsQ2fSekHpaton7YfeZ/39wKSTZX1MZX3X+irFBryCvmPDNHcNU15849Tr/OFzbp/aEr1H7iISvRfsUks+5swlFlF7aR0TMVBr5J049TzrnTteu1Jnfi9CcBp+TBwKcv1DUuG9e8pjYifyrJ3zZm/B1bZB/5bFB2sckfD1Szzzybwbq/BVjA/P89Yr8jZOS6hQ38pf8cjKxKe/b1aVbSd6qCd8hIn9z+klR5jnKeHrRvkTn716DSYxKPbqPypjuxJYjbrdypxtM+AYi/8m2X4ne9izmIgcct2BbYjcyOwAjWe2P9nETvq2ILFVRvwlf+ZbW8gzGUl7bBLSks2Js0GtNXuObjNOXDtCNTjPKIlWqGN16N+ZCEtF6C4j9A4GL6U5cKdFnHEMkkrBdJ+ruegafKApHlF3mLO+RbYvN+coriJq692awMLSP6/xlbiKU8PWt2JX0GYr8Be3jxP5Rxvmn31Paxx3kQjCDVanOv8+Er6S+5OS57DiZZG8A6vwVY4dQ5O9ORrIif0eOmEetovpkWv/FpUxCkX+vhO+iQOQrkUX+UucvOH/AoZoqSD0NlSXREZG/TC67mHHemoyzZC5J+Iaknn04f+Y8RxMzF3X+ztyE/hO+yb0r1fmLwcc6Ns4HTzjty3T+5np0GUeFYkj0nuFrc/7SWbk0TDeLWnPHNdNNI/8Az558Fn30iPwXtVvB/QZlOv+Jdh51u3b7pIUS7mBnAmQ5M9g3eMna+0BO0SSRv53v8N1DiX44/+TcZpZ18TfOB8f0OoiSCp0VnX8W+bfthK9EkPbhKjr/Ir2TU1SVTOwb6vwVY4NeUk/jjPPyDpy9chdonyxqFZF/x5Y55m3FZysKD9s6UzHyN7F/pvMXFMOkL/KXCd8Sp+JSMVUj/2kxcBLJiLsY+ceetyeJyUDkH1IqyRIbIZ2/dKhRBHT70PnHMRzO3yndMWDC1zh+95IyeW0lC/uHOn/F2CDn/AO0T/qwdQTnb5KOrvPPlCpW5B97KSX2RN7uZxfT3diSFYZQjPwl7ZOXnHDP2fI4Gwk3GpeJUllHyIWkfaSCpcD5x3lSOBT5+xK7gH82rDmnOVdohi+bN5h0EPNF/r4XC5NTkJRTLTr/WPTjnDf7v6SRv0IxHMzjF/K5kvbhlKYwHK+kfZKyyclnGfmbipYurMg/wP+7mO7EWFyF9kn/trwJ3+K6vjbHHPYq+53BzgyIhisHik4uieZT+7txOos4d7ohnb97LoNW5HfEIcpquizy90g9Q4XdfCWjDe3TbuUDWqLzF23SL6GEr/smY+owmXsUopFCk/GGhTp/xdig1wLuUudvHrxFWeTfFe3yY6Yczt8X+Yfq+ZSv+cp9Rf6URtmSSy9L+EY9In934lXuNMNVPa2lFzuxVTY6Zi7q/Ln49iQhKSZ3uzfyTwcR7wxfR+oZRSbyLw48oXkEccxZrsDXLpzwLVYETbYjXZA+2e6eNU/4NgN1/oqxQabzD3Dtuc4/T9oaByp1/vLh3u/QPr5IkjkfeHxRuA+VE77CNbTSSLY4wUoOPmnbKKzzB4pUjKzqGaJ9pHOf6cZJ5VARcbsL2fRK+LYiv5MP2W5z/n7ax5hMqRP3jb+hBeK7zNY1tYisNwxJuUmYCH/CyeFkfabX6F6q6vwViprQayFxk/uT9fl9nL/sR26f7vidPyAHnj44/ypST+EXonRxkHyCVTHyt2bqlnj/UI6DRDTu3k9Zr3+my5aCpSzhG5rhG4r8Q3SQ4fzLEr6yto+vpALgXy/ADFYy8nePl8l2iTzy99A+cT4Bz/09shm+yvkrFMMhc8A9Er4znTjjuHPOP49OyyL/kD/3KUFKdf6d/p1/i8hKLoZon0xd0kfkL/nnViDyl1LOaRP5p+cwORTZX5XIv+UxMqGsitvNm4fv9w1JPX0IRf6GomnJyN9pE8r5dOOiekm+TQAlkb/XyuGhzl8xNjDPZVDqKRK+ppKkeWBldCoduDXJq+t/+OW5Q0s6uhiI9onI4tInfbSPcDZlcOWXcrZpRuUUnL+H9hGR/3QnRPsYGaltg6RYCtvJ/i7P79ZWknAnefng2x7H+b0zu6VtZsWyUGDRiePCwvNx+pYWZbSPP4egkb9CMSR61vPPdP55iYYJH+0jI/+Ow/n3mD1sR+FhW2cqSj1lWBhVTPhmzqakW3fWrRX5B3T+LucvSxYwEs5/okVoR3ZuwpzLHeyCtI8zO7ntOn+P1FOWewZSyWrI+Xu8rVl/QNokdf7mmkKDf3Ltxcg/eZtIvhd0/iI/0QTU+SvGBtkkr+D+PPI3zn9Rq6j2kQ+49H/T3ThI5Zh2UlceHCjiJOHcT3kHIOX8rYSvf4ZviGbw2evaSiUJX1vtw5ac1HD+E62okJsw53IHu1DC150QlTn/VErKXOT83Ql+vlo62XkDnL9Z/tHY1HIGIFlUz8VMp8j590r4mvpISvsoFEMik3r2QfuYxTsMZdGKiouiG8x0ekf+VZZxNBF0tfIOjtpHRv5tE/nn7S1n04db8UX+xYSvzflLSWayNGTiALPchHO8O6+hJWSl1nZnhq9JbJvfLCns5tjv3P+I/BE+UK7zl1RUJGr7JCuWhQf0mTjOaLisz14J36yYnkb+CsVQyDn/EO2T/JWR/4TD+bei4upYBtNpRUv/uYu0D+BP+ubOv//IPxYTrXycv+VsPD4l5Gf8CV+/3YDk/JPvzMn+iVaU8uPFa1/sRP5RBG9U3IpQTvt4pJ457YPsOoK0j7OdKHljY7av3xv5h5y/h/Yxg0Uw4Wvetrw9Dg91/oqxATt/XRhnNN3JNemTjs7fRNc+hGrUADLh6zglT1/mLaOfSV7Gtl5VPcucDQBMeKJe2UeiFLK3GXQKnL9dU6eTcv4RFevdRFSs4imjbGmXS9mYRKp58/BN8nLfvGTdIReu2mciirJgwFL7RJT9Z2q37HvvYqbDhYSvyRHkb2Io7De2NgF1/oqxgY96kZC0TzbJK42+TVQbUVii6VbClPBN8pLnlDADTz/lHYCi2sfP+ctyAUWv4joo185Snb+Qcna6bBWPM1U988jfdpRmu0RLnEva5ZamMDRNmdrH2M+cX0fVyL/doqzvZEBKtsvj21EU1Pkb29x+szyCoJF8Nmt5B4UigGvu2YqbHnqqd0Mn6edClncwUayb8A2txwpUi/zdhV58A1F/kX/uGKKomtonU5d4fIpP4w7Yk8NCi7lYJbC7pryDkHqmkb+poy8d5WQrKji5SOj8pV2hyN/W+feifeDNJ/juQTsiMfjbtI95x2i3ytU+Jgfi2iQjfxdV1xoYFLqMo2Lk8aZLbgAAPPiRXyxt10/kbx52t7xDHPupGiA8Wcmck5kLC4b7HvBd+zsAgCWTvR9PK/I35R3SPifbxQjdTvgWEVpAxZQ+Jsuh27a7nP+yRW1rZu1MJ1X7pPSUvBUTbX/kn9E+hbVzyfleXtLZXcWtbJKXa8dEKxKRP7yRep7H8HbpjfxNVc9QDmZkZ/gS0elEdB0RrSeidUR0VrqdiOgTRLSRiG4lojObskEx2nhg2x7c+/iu0jZP7p6q3F/mEEJqnyzyz3X+bnmHLod53VCBMiDlvH2zPz3OYsOjOwAAzz5yebC/DMIxZBJKh/aRTrZXwjdI+3gi/yLt45Z3kIXdkv2TqZN3k6MTrSINY0X+Du1jRf6Z88+T6gW1j6PzLyvv4Kp9EtqnmPCWtX0yGwLe35S4ljC0T67zd/ZnM3yb8f5NRv5/BeBDzPwfRPSa9PtLAZwPYE367wUA/iH9q1BkeHL3FF72f36IJZMt3PHn5wXb3bZ5h/WdmYPSOF/kz8z4+x/eh6WTrby8QzfOqj0a52+omNDCHEC4Rk1ynuLyjoD/LWL9ph1YvriN4w9dmm0LLTnoFnaT5R189IyV8PVx/j0SvkRhtU+hqmeA809KKdt2JRJQuz+Z8G07CV9b/RNl5zTnKkb+uR3udbgocP5RhL1TM9k+OcmrYINvvcbUtiKnDzvh60b+DSd8m3T+DMCELgcCeDT9fAGAL3CSebmOiFYQ0ZHMvKVBW8YSzIw//Ndb8frnr8KLTjxkrs3pC9+4eTMAYO+0P5r+1FUbMdON8fEr7wWQcMZr//eV2LZ7Cq8742j87stOxDdv2Yw/fNUzs8FABv4798/go9+9GwceMIFP/GCj1XdS2M2O/G94MM8pSOVHqM6Pi3d95RY864hiJP+mS67HqUceiCMOXIyXPnMlzlh9EG59ZDued8wKyzm1iND1vLLYTjDn0mVdnMIkr4CzAcJLJ9rlHZJthfIOrs7fmuTFFucvcxNA8vv5E75Fu9zJXxMO53/Z+kdx8NJJq68Htu3BX3znTvzyc49Kr8M/exgovv1MtHLOX547ijyRf8cfGJgciEQh4euY80/XPpDZ2gSadP7vBvBdIvo/SOiln0+3Hw1gk2j3SLrNcv5EdBGAiwBg9erVDZq5cNGJGV+/+REcvWLxyDn/3VMJ733iyqXe/Zeu24SHntybfZ/uxtiWUkB3bNmJK+54Ap+66j689ZwTsXzxBIBc6REz43e/eDOu3bjN27dV3kGEd695zhH4zm2PZdHdRJrkm2xHiGMuTfhed/9TuO7+YlL69s07cfvmnQCSweOM1Qdh01N7sfbYgxxKB4BnbJFuQXLpUiljJXxZTipKcNrRyzMbQqtnDVLeoUWymmaeB/DRPu1WcTZvFOURtbTLXYXM5fwffmovPveTBwEAqw46AI88vQ8AcPE19+OklcvSY0qqehbUPjnnb5V3ICoc4+Z0rPsRET7yuufg+geewjdv2eyRevrtmZecPxFdSUS3e/5dAODtAP6AmY8B8AcAPtNP38x8MTOvZea1K1euHMbMsYX5DztdcZ3S+QSZvPPuFxTLyYcvyz4fsXyxlaCT6ho5xf+BbXuy7SuWTFgKj5mOoH1EFHjUgQekttkDw2QrwkQrKtTD8eHXzlxViEqz82aLnzMWTbTs4mUBD+CL/BMHnzu3bjDyT/6+7oxVeOvZJwAoUft4J3mFOf+YbW7e1PNPVD1F1VRI6mnuv6X2cSZ5mTkA0hxD0/3PX3w23vD8Vdn2q+5+AgAw2WpV1vm3o7zuv1XPP8rVPu4A5CLm5Pe48KzV+NgbnwcgV/uU5WCA5iZ5DRX5M/O5oX1E9AUA70q//iuAf0o/bwZwjGi6Kt2mqBnmFTT0H3I+Q64F64Mc0KQqZslky6rH7y4gAhSlnhOp8+7E3ewYl/YBgMUTLatPMwHMTFwqo30MJtv+ejWyX1MAzZZxBpw/7DZdzh18RvtIzj8u9sWi/1DCtyP4Z3dJRNf+zB5H6mnq2ySzpN2Er8f5R5QNsEWdv93ORV5B1KZ3fnj31vR8doL5n960Fp//6YP40b3bvDr/7FxEWW4i8kT+Zc+aOY7SRLihfUJ5FoN5Gfn3wKMAzkk/vxzAvennywG8KVX9vBDADuX7m8G0xwGOCozzLXuNNlgymU+GWrKohWkh1bTW3s3+Fvlm+YBLnb+kfUz5AVcGagaPMtrHIHFy/n0zaWG4TmwSo/m+ED9tz/BNa9Ck0WSQ9iH7WBbyz1bAEVnLOAYjf/t7MeEbY6JtEr7F38AdFIly5y/tkvMHAP+AZX53OVgBwL50gJ5oR1bkv2LJRPb/qKD2kclmcV8jygMJ01fZsybtkPcgn3Ed8vKjx/n/DoC/JaI2gP1I+XsA3wHwGgAbAewF8OYGbRhr+KLfUUFO2/Tp/Cfb2LFvxvvWkys+7O0TLbIW2pjpMmbiksi/Yxd9Sxw1V4r8TWEzH6ZlQblWVFDy+BCifexa+g7t41H75GUUyiN/i05ynP+0o3aSNrBI+Pp1/naEbj5nA2xk2yqv20dVTXXzipi+ezfpDMJE+TmLOn9pl6jqKdplE80CCV84duSyXBRyMC5GTu3DzNcCeL5nOwN4R1PnVeTIOP+S/5DzFb1oH+m8DxC0z9LJFmZEbZ4Zi/Pn7O+05fwjK8KfDkT+5rPL+ScUArC/ROop+whRODPdfLWryVZUWKIR8Ek+nWiS8yRiHvnnrZPI3+aYmSEif79tcuH3Kjp/09aifVLO36/ztyN/Y4+ZqOZSL3bkX3xbySN/v6qn+DvkfRY5f1tmmkX+TiTvuwcS1uAmZLk5jeQ/Tss7KPrGaEf+YdrHXRJwyYQd+Yc4fxaRv4xSJ1pRFsWbY8xxsrKmedgztU9k0z5lk7wMJlthiWGyfGSuJLIpHb9zdiN/w6Un9ESyvZDwdSJNRj7RKORo5ALuIZ2/O4nNSvjGiYIqr+fv0fkLb2T8raTW8n3+ev4+UOCaJlq2zr8s8nfzDb5lHN3Koj64M4IznX8P2qehwF/LOyxkTI9wwtc4fR/t4zqZAyzax+H8pfMXOnkr8m9HmOgI2qcTZ+UcFouBxTiLjPMXtE+Lh6d9rLIS7cibUJyICNPiGNmTWWvARP6ylr6BT+fPgnoI5R6z2aaUFzYrlHfoFCN/uZhLVtLZ5Cac2j7y9zE2Zglfx3FatE8oiQLj1Ivb3QSzajV4TAAAGKlJREFUpIeKOn9/vsGK/CslfOWbglwgxv49fNfQBDTyX8AwcsUQdTKfkc/WLJcUtiOyePmli9roCPpkxlp+Me9D+q1JD+c/3Y1BZC+6bT4aJ5dFpe0+Er7tMO0zLVRGCeefwzjcssg/SqmEbmxHqEWdvzk25eORO6bekX81qWdia05lMOfLOGa0jxX521SOW9OnoPOXtE9Z5E9+pZRLM8kqn8UZvrZdVuTvTPIqk1W7OY1M6pnp/APX0FDsr85/AWOUaR8Z3bv2y1mUCeWSPxwHGKlnx8P5B4r6+Dj/6U7CT8NxrtKeXOdPmGhRJZ1/r8jf2O2qX9xI2KCwgDsLnb9x0iydP4q0j9wWsK3fqp7GnlwWCru8Qw+dv/lsdP7yN44imx8vp338ZRwmHPrNivyd159Q5N9yBgXADjZcuG9ypg5T1k+FpH6dUOe/gDHKtI9dJMy2307WUvawtiLConaETsxZCWaf2seFHEDMg7ZnuoPJAPVS1Pkbzr/3fZ70FDDLr5OtgSWU8JVwI39ZLyZLzDq0T3ZNJipHvi3kR01VT0PlEPnr+bv2mP5m4hjduETn70gvi7SPO8NXRP6BuQnmGr0JX+cNTL4h9Ob80+uLCEZAXIXzt3IaIuEbKu8gbWsC6vwXMEY58pf8sUtbyeuZbNuRv3EWpiaQlTAOOv98ADELqOyZ6mJRuxUoI8DZceack+1qM3x76fzzOQQ2tRGO/KV9OZdu6fzZpn1cqSdz7mBCtI9bXtiUj5aY7sbWAjRy0ZVpIY+VuQmDSUd9k9E+7Qqcf8kkKQpc06TzBkbInXqp2icKlHfoU+dv1WHKxuJQ5K+0j6JPLITyDu5n97tL2Uw6zt9S+5TQPub5NhO5dk91sMiJ/CPXkTmRf2iRGIl2mc6/Izh/Z/3ekBLFGiAM7dNnwhfIB5Fg5C84fyDXqUvMdGJrHV5pw5RDZyWDVH6sq77JBruMEsmdPjmcf0g9lVxjWOpZ0PkHFFUTAZlpKxKTvKpw/s7/peS3Ktbzd4vrNRT4q/NfyMgi/wr68/kGGe33cv4mGUjIX9H3THcKbctoH/PwGnXPnqmE9nGllLJP46An2xSshlk8VxntExcoJYOQBt1tE4skomnadWkfTx+UDQiByN9x/kan7tov1VGRkHoaGay51+66CG0n8ncneZm5CHliOj9v2b0n+AcHtzKosRee9m0n35DP8A1Tgj64OQ3zW7mL67i/vdI+ir5hopBRpH3chUEkpq2EL1l8fUb7TKWRv2jLgdBcKj8s5+/y7umXbH1f4aDK5IYSvjIGBjNdzmcPu86/otonT/hK2idvE3tW8pL3JTS2yEVQjB3uf6uZLheksW7kX1bPX5470/mnA2wntpPY9qBcLvUMOU9b7ROO/N1JXjLha64gc/4lgVbxLc1ZXyHdvWjCXrtZ1T6KvpErXkbT+ZuHoSrtQ8glm1nkH/eO/CfblEXUZlLXrqkOFk1EBTUN4Kd93GgthMo6fyeazTXoPdQ+Wb0YUYDNpX0cZ1OJ8xc6/6SdfxlHd1Kc6S53/mTlJgzcyW9ujiOjsoxMVVx3eeRfbdEWEgXgfFU95TE+nX+1wm6S9kEh4WuuSSN/xdDoVSJhPmOmy1ialm3oSfuIh9BMvNpnOH9PYTcDSS14aR8RjRLlag2X9nHlpmVIVCb+fbKaaDHyN7aWR/6GS5cUiSv1dJ0NQ3L+/uuQM3yNPT6pp0X7EETkn/we2TKO7Or8nfyKI/XsxLG1rKP0z+Wcf3h/WO3jFHZr2ZG/+Zro/NmyoVznb9uc1WFy3mYm2+r8FUMiT/iOZuRvZu4WaB9H6pk5SkH7GGflq+1jYArC2bRP+uYw1U05/zwadHX+JkKblDb0gKsvt6/ZlnpKBMs7yDZRvsZwqKpnV9SSkZE/XAfknD/2cP6FhK8T+cso2chgswXcnci/oPN3In/zxhJFjvEYrLyDPIfpriV+awl3FTFf8n2gqp7sr+pZTPgq7aPoE6PO+Rvn7JZ4kA7dpn2KTtMq7+AEZebNQjqeRalUcbeT8G1HkXi1L0o9qzr/cs4/Fs7fbpNX3QxHhYaHN9GkrKhpYC0ekm5jcCHyd8/fcZx/FHkSvh12In+R8BVUWSuitLRBfuxE2z/JSw7mcu6CHfmX3HsR+bu33VL7oGyGb6C8A3k4/4q0TytK3hpiFnmMdN+itsP5a+Sv6BejrPaZ7sTZIi3um4scDCbbdi1+N2L1LeZisHRRKz0mf/ClVDGReiaf20KKWKjn344Kr+oh+BYtkba6+QSDcGE3O5pk9tXzz9tLmgHZ4CA5f2TXJJEtfC4ojwLtEztSz8ij80/vo0v7TLrlHcwg1M4jf0n7yLvQi/PPflvHqYYSvu6ksZDaxzfDtzNAeYfsxTVE+wR7HA7q/BcwjJOcicP/IecrZrqcRf5lk7xcvt11mr6qngZLF4nI31H7AMBkuwXz6LUF5eAqcvri/EsTvnmpadcBZMnIEg24LO/QklLPnlU9c2ohj/zt8xeknhEVEugu5y+remacfyrpjD1qH5cTT9rnTjVJ+Np2yrY+SKcuCwC6xxHyN4FSnX/k1/lXqurpDG5G7ePq/Iucv9I+ij4haZ+QzHG+ohOHaZ9pi/bJZ+cSFZ2jHDjcyD/j/AXlIKNDmfBtt6LCq715SPvh/CfbYZ0/kCeqQwnfAr8tvrqrQ2XLBRYKuzk0COdlGYyjcd+guoWEr6e8Q4dttY9P6pmWcfDV83cXO5H3wVyTb4JaaVVP5E59cdt/T01/oXvs0j4+nX/f9fwjWc/ffkNb5NqptI+iX5j/iMzhhdDnI0y9/jzydwu7hWf4Fjj/EspLcv7mOZZ1/cMJX5vzb/fB+fsif+ls9kwZ52+3MV97ST2N2sda/CW0gLtX7WPstM/vm+TlU/tMCqqsJXT5ks7KVrFy1T5WAjXfDhi1j+Tv87blVT3zfYvdyN/6HUhQa/Y9tovK2QlfcwV55B9+zuzrE+UdMsrIP8FPE76KvjFTUh9nPsPYmnP+JbV9BOUidf6+tm6kamiAScHnS4dQ4PwzOd+QOv8CrZAfuzedn1Dg/ANRaSHhy+7qUNSXzt9nkzlOHuMr75DV64/ye5nr/LuZ/V6df5sKCVFpR7ZGQS7sylAWGRPlEbXL+Vu0D4XvsVtOWiZ8Mxuc/xs+2G82yTWxSPj6lg5NjAtf3zBQ57+AIR3fKMk9jd1htU+gvAPZkbvbNqT2kZSOjM4WtfNJXu0oj9hDJZ2rwLeMozx2T8D5Z+UdClGh3cZw6TI6tev5F9UlidonfwtoRUUaKy/pnDt2X3mHZFZ0fi/NtWa1fQzt49P5O1LIpH0aFTsJX7cWfwgkbJfJ6EIf8toKtI89KMncgKFT+63q2Yoom4BYiPxnSeevK3ktYMiIeZTknq7zL5Z0Fk7DqatTTPiWcP6LhM5fRKsGttQzjzp9JZ2jqNqbVaJ2cbeJyH+qa2nJDapF/sjLOwT0+D51CQvex9TQcdU+RsUiHWSR9mHrWmTS2dL5RznlYSCltPb12pE/HNuB8sCYKM95LJ4oi/z9+n1js4EcgHz29pPwNW1z559G/iUDfJ1Q57+AUVYZcz5jOnP+vWmfdmTr/N0Hx17G0UbO+YsHOiAbbUV5YtnUCzLJ5YlWhKhiTsUf+efn2TPd8b5FmFIJbhRocf5OwtccZ9E+VsK3yPkzkho6E46Nbm2fiMgaTM152w4/7s7wNfc6SXbm/feifToxJwNfnJ8/O09paJwPUgcUnL9slb/1uf0VFo8vURzJWlIu3MHCtM3yIsHIvxn3r85/AcNy/iX/KecbTARkdPgF2kfkMibbeXkHoiJdYZV3cG7Bkkznb/PUBosmosxxSCmiqTuUOah2hCiuNrj6Er6Sqto73fUmj025Btcxya/GKcmJXG7CV6pLDBLO34T+aeTv2NDJOH9/5O+bmdyKirV9zL3ulfB1aZ9uzMlg7Iv8e3D+Mfsj/5DO380h2PX8/XMu+q7qaUX+ybZugPNvSu2jzn8BY9Q5/wMq1faxHVVR6imdvzPJS6h9fHzvZCvKnF5bTEKa7saW5G+yRehQ0WH74CvvYEX+aU0hF6a0gesH5PeWYx9g5J95G18VSTnDl5EMIoVJXjE7VTfJqhbqK0UdrOoZpbmJgtTTvl7AoX1E5O9ObguBEHb+xcJuxe2Jbf7IX67h64oBfHATxKZtVi02FPk3RPyo81/AkKWPR4n2yTj/iVBtH395B/Pd7stOdkoYGiDR+Sfb5AM62W5lD2Rb8PAzXbvIWCIV7f1mZVbncvXl0mGGIn+zcHrB0XkUJzPdWPDXOefNbKtLMqfCkv9PnOykM4h25PKPSKSYsRX52/JXY0+B9ilJ+LY8kb8ZzN11CKR15ZF/PviVJnxJlK4o0D7220xOqeVtqtT2CUf+6bHp/Vjkcv4NRf5DqX2I6A1EtIGIYiJa6+x7PxFtJKK7iejVYvt56baNRPS+Yc6vKMfIcv4dI/UM6PwLah/jzHpx/raDNnSLXGDFivzbOe3TjvK3gxlntmlVqadx6u6KVQXOv1182luRTaMYWJG/0Jr7Er7mWoqRv/3ZR/vErvMP0T6ysBvllEWe8CVvwtddxtGc3kT+nThO+sv4dnEPKkb+Rc6fvN/dnIxMsrsJX/N/yq375IN7/0zbPF/gj/ybwrBnuR3A6wBcIzcS0SkALgRwKoDzAPw9EbWIqAXgUwDOB3AKgF9P2yoawKg6f2ProokofUjCzn+yFeXFzqg4OcnS+cd2FJVLNSMv37uoHWXRmNT5z3Ria7Zp1cJumfN3nIm0ee9UgPOPEuWM6+gstQ9J+/L+TYRtnG2hvAMLqScbm4qcv5VfcFREvppEsrZPRvsIyWzXUW35Er55fSIzw9dcdz4QlgXGRPmgt8iJ/EM6f5djj5zB2qcMcx24D+7vbtq6iwTNVmG3oWgfZr4T8I68FwD4CjNPAXiAiDYCOCvdt5GZ70+P+0ra9o5h7Ahh+95pvOHTP22i65HApqf3gih5oH//X9ZnkfR8x74ZMyEomcD15esfxvc2PJ7tf3zn/nQxkZRKSZ/ppZPtwgO94dGdeOXHrs6Oa4uIS07SMlGhjPpkBLZkspX1vWuqg+WL25kyaLJNqLB2u9f5L1/chlnFKmbgqb3TOHTZosKxhm5wqQvJB0fCPuk8r7jjcbzyY1cLxY6xO+lLzlA2CXTX+e+b6RaKtt388NPZvZXVSDNFUJR/3rl/JstbmK6Nnd2Y7clhgi4yA+NkgRZKnG+H8yUrfYIruUiLG/m7cwVCUk83sWveBKKIMkdt7qW8pnZEmUNPjhV9UtJW9p/Z6TynTVVmaYrzPxrAdeL7I+k2ANjkbH+BrwMiugjARQCwevXqgYyIIsKaw5cNdOxCwJrDl+GlzzwMNz7wVDZ5aFTwwhMOwfOOWYHfe/kabHh0h7VvzeHL8PxjD8au/TM495TDsXzxBN573jNx3qlHgIjw7nPXYNNT+/CiEw/BD+563Drulaccjmvu2YZTj1qOF514CN569gl49pHLcfLhz8CuqQ7e/PPHI44Z92/bg7XHHoQVSybxuy89EW95yfE4aMkk3vzi4/D4zv143qoV+LnjDsZbzz4Bpx19IGa6jDeuXYVDli3CkQcuxhnHHIRdUzPYsn0/7t+2G885egWe3jsNAPjVM47GsYcswZ6pLl516uF4YOse/PpZq/HT+57ErqkZvPTkwzKb/+r1z8VxhyzFRIvw7COX41eedxQmWxFOOmwZrr5nK1516uFZ25c/6zD8bNN2dGPGa09PHrff+YUT8JP7tmVtTjnqQJz77KT/Xz9rNR7bsR/vfNlJWNSO8LZzTsTbzjkB39vwOE5YuRT/5QWrcc/ju3DfE7vx6I59OPWoA7N+fvOFx2LFkgnrdzlj9UF40QmH4gOveTbWPfQ0zl6zEssWtfHWs0/Apqf34llHLAcAvPKUI3D347sRM+M1px2JjU/sxiFLJ3HOM1finS87CccfuhRHrlgMAFixZDL7be/buicbwN70ouPwy887Cnc9tgsvPukQ/NkvnYL1m7bj3FMOx859Hdy/dTd2T3Ww5vBlOOmwZdixbwZveckJWLqojbOOPxgAcOwhS/BfX7Aa7YhwxPLFePGJh+LtLz0RZx13MP7o1c/EiSuXYfP2fTj1qOXZb7t4ooVXPPswbN87g0OWTuIvf+25+PxPHsT5px2Jt7xkO7ak9ykiwgkrl+LhJ/di30wXDz25F2ccc1B2r96w9hhMp5PiXnLSoQCAd5+7Bu0W4Y1rj8He6Q5WH7wUDz25p5CorgvUq+AXEV0J4AjPrg8w82Vpmx8C+ENmXpd+/ySA65j5i+n3zwD4j/S485j5t9PtvwngBcz8zjIb1q5dy+vWrat8UQqFQqEAiOgmZl7r29cz8mfmcwc452YAx4jvq9JtKNmuUCgUillCU2nlywFcSESLiOh4AGsA3ADgRgBriOh4IppEkhS+vCEbFAqFQhHAUJw/Ef0qgL8DsBLAt4loPTO/mpk3ENGlSBK5HQDvYOZuesw7AXwXQAvAJcy8YagrUCgUCkXf6Mn5zwco569QKBT9o4zz15LOCoVCMYZQ569QKBRjCHX+CoVCMYZQ569QKBRjiJFI+BLRVgAPDdHFoQC29Ww1+1C7+oPa1R/mq13A/LVtodl1LDOv9O0YCec/LIhoXSjjPZdQu/qD2tUf5qtdwPy1bZzsUtpHoVAoxhDq/BUKhWIMMS7O/+K5NiAAtas/qF39Yb7aBcxf28bGrrHg/BUKhUJhY1wif4VCoVAIqPNXKBSKMcSCdv7zabF4InqQiG4jovVEZBa9OZiIriCie9O/B/XqpyZbLiGiJ4jodrHNawsl+ER6D28lojNn2a4PEtHm9L6tJ6LXiH3vT+26m4he3aBdxxDRVUR0BxFtIKJ3pdvn9J6V2DWn94yIFhPRDUT0s9SuD6Xbjyei69PzfzUt64609PtX0+3XE9Fxs2zX54joAXG/Tk+3z9r//fR8LSK6hYj+Pf3e7P1i5gX5D0nJ6PsAnABgEsDPAJwyh/Y8COBQZ9tfAXhf+vl9AP5ylmw5G8CZAG7vZQuA1yBZhY0AvBDA9bNs1weRrBLntj0l/U0XATg+/a1bDdl1JIAz08/PAHBPev45vWclds3pPUuve1n6eQLA9el9uBTAhen2TwN4e/r5dwF8Ov18IYCvNnS/QnZ9DsDrPe1n7f9+er7/AeDLAP49/d7o/VrIkf9ZSBeLZ+ZpAGax+PmECwB8Pv38eQCvnY2TMvM1AJ6qaMsFAL7ACa4DsIKIjpxFu0K4AMBXmHmKmR8AsBHJb96EXVuY+eb08y4AdyJZk3pO71mJXSHMyj1Lr3t3+nUi/ccAXg7ga+l2936Z+/g1AK8gInsV9WbtCmHW/u8T0SoAvwjgn9LvhIbv10J2/kejuFh82YPRNBjA94joJkoWpweAw5l5S/r5MQCH+w+dFYRsmQ/38Z3pa/clghqbE7vSV+wzkESN8+aeOXYBc3zPUgpjPYAnAFyB5C1jOzN3POfO7Er37wBwyGzYxczmfn04vV9/Q0SLXLs8NteNjwN4L4A4/X4IGr5fC9n5zze8hJnPBHA+gHcQ0dlyJyfvcPNCdzufbAHwDwBOBHA6gC0APjpXhhDRMgBfB/BuZt4p983lPfPYNef3jJm7zHw6knW6zwLwrNm2wQfXLiI6DcD7kdj3cwAOBvDHs2kTEf0SgCeY+abZPO9Cdv5li8jPOph5c/r3CQDfRPJAPG5eI9O/T8yVfSW2zOl9ZObH0wc2BvCPyGmKWbWLiCaQONgvMfM30s1zfs98ds2Xe5bash3AVQBehIQ2MUvHynNndqX7DwTw5CzZdV5KnzEzTwH4LGb/fr0YwK8Q0YNI6OmXA/hbNHy/FrLznzeLxRPRUiJ6hvkM4FUAbk/t+a202W8BuGwu7EsRsuVyAG9KlQ8vBLBDUB2Nw+FYfxXJfTN2XZgqH44HsAbADQ3ZQAA+A+BOZv6Y2DWn9yxk11zfMyJaSUQr0s8HAHglknzEVQBenzZz75e5j68H8IP0TWo27LpLDOCEhFeX96vx35GZ38/Mq5j5OCR+6gfM/F/R9P2qM1s93/4hydbfg4Rv/MAc2nECEpXFzwBsMLYg4em+D+BeAFcCOHiW7PkXJHTADBIu8S0hW5AoHT6V3sPbAKydZbv+OT3vrel/+iNF+w+kdt0N4PwG7XoJEkrnVgDr03+vmet7VmLXnN4zAM8FcEt6/tsB/Jl4Dm5Akmj+VwCL0u2L0+8b0/0nzLJdP0jv1+0AvohcETRr//eFjS9FrvZp9H5peQeFQqEYQyxk2kehUCgUAajzVygUijGEOn+FQqEYQ6jzVygUijGEOn+FQqEYQ6jzVygUijGEOn+FQqEYQ/xf2g+2I8Iy68kAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u2HaGRVEYGQS"
      },
      "source": [
        "## Testing\n",
        "The testing result will be the average reward of 5 testing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 286
        },
        "id": "5yFuUKKRYH73",
        "outputId": "3ad20d21-4c56-47b7-e617-d71e4211aed7"
      },
      "source": [
        "fix(env, seed)\n",
        "agent.network.eval()  # set the network into evaluation mode\n",
        "NUM_OF_TEST = 5 # Do not revise this !!!\n",
        "test_total_reward = []\n",
        "action_list = []\n",
        "for i in range(NUM_OF_TEST):\n",
        "  actions = []\n",
        "  state = env.reset()\n",
        "\n",
        "  img = plt.imshow(env.render(mode='rgb_array'))\n",
        "\n",
        "  total_reward = 0\n",
        "\n",
        "  done = False\n",
        "  while not done:\n",
        "      action, _ = agent.sample(state)\n",
        "      actions.append(action)\n",
        "      state, reward, done, _ = env.step(action)\n",
        "\n",
        "      total_reward += reward\n",
        "\n",
        "      img.set_data(env.render(mode='rgb_array'))\n",
        "      display.display(plt.gcf())\n",
        "      display.clear_output(wait=True)\n",
        "      \n",
        "  print(total_reward)\n",
        "  test_total_reward.append(total_reward)\n",
        "\n",
        "  action_list.append(actions) # save the result of testing \n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-207.9114975585693\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAfbklEQVR4nO3deXRV9b338fc3AwmEMAdImBLaUGSQKUK01CKrXi1V4fqwFHutFvXSKq1Dr4reZ7WlvbUWa/Vely590Dq2FyuK1NV6L1pEqlUqoBEBUQaZwhAgTCFkOMn3+ePs2AiEjIeTnXxea+2VvX97ON9fzs4nO7/sc465OyIiEh4J8S5AREQaR8EtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIhE7PgNrOLzewTM9tkZnfF6nFERNobi8V93GaWCHwKXAjsBFYCV7n7+hZ/MBGRdiZWV9zjgU3uvsXdK4DngakxeiwRkXYlKUbH7QfsqLW8E5hQ18ZmppdvioicwN3tVO2xCu56mdksYFa8Hl9EJKxiFdyFwIBay/2Dts+5+3xgPuiKW0SkMWI1xr0SyDWzHDPrAMwAXonRY4mItCsxueJ294iZ/QBYAiQCT7r7ulg8lohIexOT2wEbXYSGSkRETlLXPyf1ykkRkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iETLM+c9LMtgJHgSog4u55ZtYD+AOQDWwFrnD3g80rU0REarTEFfcF7j7a3fOC5buApe6eCywNlkVEpIXEYqhkKvBMMP8MMC0GjyEi0m41N7gdeM3MVpvZrKCtj7vvDub3AH2a+RgiIlJLs8a4gYnuXmhmvYHXzWxD7ZXu7mbmp9oxCPpZp1onIiJ1M/dT5mrjD2Q2FygB/hWY5O67zSwTeNPdv1LPvi1ThIhIG+Ludqr2Jg+VmFmamaXXzAP/BKwFXgGuDTa7FvhjUx9DRERO1uQrbjMbDLwcLCYB/+3u95hZT+AFYCCwjejtgMX1HEtX3CIiJ6jrirvFhkqaQ8EtInKyFh8qERGR+FBwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGTqDW4ze9LMisxsba22Hmb2upltDL52D9rNzB4ys01mtsbMxsayeBGR9qghV9xPAxef0HYXsNTdc4GlwTLAN4HcYJoFPNoyZYqISI16g9vd/woUn9A8FXgmmH8GmFar/VmPWgF0M7PMlipWRESaPsbdx913B/N7gD7BfD9gR63tdgZtJzGzWWa2ysxWNbEGEZF2Kam5B3B3NzNvwn7zgfkATdlfRKS9auoV996aIZDga1HQXggMqLVd/6BNRERaSFOD+xXg2mD+WuCPtdqvCe4uyQcO1xpSERGRFmDupx+lMLMFwCSgF7AX+CmwGHgBGAhsA65w92IzM+BhonehlAIz3b3eMWwNlYiInMzd7VTt9Qb3maDgFhE5WV3BrVdOioiEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGTqDW4ze9LMisxsba22uWZWaGYFwTSl1rq7zWyTmX1iZhfFqnARkfaqIR8WfD5QAjzr7iOCtrlAibvff8K2w4AFwHggC/gLMMTdq+p5DH3mpIjICZr8mZPu/leguIGPMxV43t3L3f0zYBPREBcRkRbSnDHuH5jZmmAopXvQ1g/YUWubnUHbScxslpmtMrNVzahBRKTdaWpwPwp8CRgN7AZ+09gDuPt8d89z97wm1iAi0i41Kbjdfa+7V7l7NfA4/xgOKQQG1Nq0f9AmIiItpEnBbWaZtRb/Gai54+QVYIaZpZhZDpALvNe8EkVEpLak+jYwswXAJKCXme0EfgpMMrPRgANbge8BuPs6M3sBWA9EgNn13VEiIiKNU+/tgGekCN0OKCJykibfDigiIq2LgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZOoNbjMbYGbLzGy9ma0zs1uC9h5m9rqZbQy+dg/azcweMrNNZrbGzMbGuhMiIu1JQ664I8C/ufswIB+YbWbDgLuApe6eCywNlgG+SfTT3XOBWcCjLV61iEg7Vm9wu/tud38/mD8KfAz0A6YCzwSbPQNMC+anAs961Aqgm5lltnjlIiLtVKPGuM0sGxgD/B3o4+67g1V7gD7BfD9gR63ddgZtJx5rlpmtMrNVjaxZRKRda3Bwm1ln4CXgVnc/UnuduzvgjXlgd5/v7nnunteY/URE2rsGBbeZJRMN7d+7+6KgeW/NEEjwtShoLwQG1Nq9f9AmIiItoCF3lRjwW+Bjd3+g1qpXgGuD+WuBP9Zqvya4uyQfOFxrSEVERJrJoqMcp9nAbCLwFvARUB00/zvRce4XgIHANuAKdy8Ogv5h4GKgFJjp7qcdxzazRg2ziIi0B+5up2qvN7jPBAW3iMjJ6gpuvXJSRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIdOQDwseYGbLzGy9ma0zs1uC9rlmVmhmBcE0pdY+d5vZJjP7xMwuimUHRMLALDHeJUgb0pAPC84EMt39fTNLB1YD04ArgBJ3v/+E7YcBC4DxQBbwF2CIu1ed5jH0mZPSZmVnj6dv36GsWPEcoFNdGq6uz5xMasCOu4HdwfxRM/sY6HeaXaYCz7t7OfCZmW0iGuLvNrpqkZDLzh7PV8feRGJCEvu+tJnNm99B4S3N1agxbjPLBsYAfw+afmBma8zsSTPrHrT1A3bU2m0npw96EQB++cvvMW8ejBgBw4ZBVla8K2oOC0L7Rgb3mETPTrlk9h1GQsLpf+QmTZrE009/hSlTYPhwGDoUEjXKIieo94q7hpl1Bl4CbnX3I2b2KPAfRC8f/gP4DXBdI443C5jVuHKlLRs5cjCZmTB5cnR5925Yvz46/7//C5s2gTvs2QNVdQ68tQ79+o0kf8z1DO4xmdLK/Xy4/XlWv7+Q6urTF56RkcH48SUMHx5djkTgnXegshJ27oTFi6Pthw/D0aMx7oS0Wg0KbjNLJhrav3f3RQDuvrfW+seBPwWLhcCAWrv3D9q+wN3nA/OD/fW3o3zOglG9rKx/XHVfcEE0tKuqYMkSOH48Guy/+1386qxLSkpnvpQzkcz0UZRWHuCDbb9nxXtPc/z4oQYfo+Z7kJwMX/96dN4drr46Or92LXzySXT+2Wdh796TjyFtV0PuKjHgt8DH7v5ArfbMWpv9M7A2mH8FmGFmKWaWA+QC77VcydIeVVdHQzsSgdJSOHYsGt6tTUpKZ8af8x3GDv4OiQkd+HD7f7PivacoLT3Y7GPX/OKqqoKysuj34Nix6PdG2peGXHF/FfgO8JGZFQRt/w5cZWajiQ6VbAW+B+Du68zsBWA9EAFmn+6OEpHa3KMTRIcGCoIzbskS2LIluq64uHWGVYcOaZyT9y+MG3wNSQkpFGxfwLvvPdmk0K75PkQi8MYbUFEBhYXwyivR9SUlrfMXl5wZDbmr5G3gVLekvHqafe4B7mlGXdIOlZTAn/8cHf6oro6O4e7bF++qGq6qqoIjR/ZQVV3BgWOfsvmzv3LsWDHduw/g2LFiKiqONeg4BQXw+OOwbVv0+7B9e+v8RSXx0+B/TorE2vbtMHduvKtouqqqStatf5UkS6GkbB+HDu1i3Lgr6Nd3FLuL1rJy5YIGHeeBB2DVqhgXK6Gm4BZpQdXV1Wze9hZf/vL5DP/KFLK6jCU9JZPUpG7szFzD7t3r4l2itAEKbpEWYpZAbu75DB86hcz00fTsOIQES8LMyOkxiUMjtrF//xYqKzU4Lc2j4BZpAenpvcnOHk/e0O/Qv2v+54FdIzWpG5ndRjNw4Dg2b347jpVKW6DgFmmmzp0zOG/CDQzKOJe+nUeRYCf/WJkZfTuP4kuDz2P79tW66pZm0du6ijRRQkIiffoM5av5/8qoATPISh93ytCukZrUlaxuY8nOHn8Gq5S2SFfcIk2QkJDEsGEXMTz3EgZ1+yqdknvVu49ZAn06jyRnUD47drxPWZlesy5No+AWaaTOnXsxcsQljMz+P2Smj8Gs4X+4piZ1pV+PcQwcOI5PP30zdkVKm6ahEpFGysoayVmDvkVm+thGhTZEr7oz0s4ie8AEOnbsGqMKJewSEhJISqr7ulrBLdJIBw58xpGyXThNezljx6QeZPUcQ1bWyBauTMIuISGBQYMG8atf/YoRI0bUvd0ZrEmkTSgu3sGOPas5eHwL9X2C1KmYGRmdziJnUD6pqV1iUKGEUUJCArfeeisrV67k9ttvJ/E0b8Su4BZpJPcqNmx4nc1FyyiLNO1d/zol96Jfz7FkZQ1v4eokbJKSkpgyZQrLly/nF7/4BRkZGV94DcAp9zlDtYm0KYcP72bjZ8vo3W0og7pObMJYtzGg67mMHbGPo0f3sW/fphhVKq1ZXl4eDzzwAPn5+SQnJzd4P11xizTRtm0r2VX8PmWRw03aPykhhayuYxg8+NxGB7+E26BBg5g3bx6LFi3ia1/7WqNCG3TFLdJkFRWlbNn6LpndR5Pd7fwmhW+XlP4M6JPHZxkrKCraGIMqpbUwM/r06cMtt9zCzJkz6d27d71DInVRcIs0w7ZtK8nJib7UvWNy9/p3OEFSQip9u4wie9B49u//LAYVSmvQoUMHpkyZwuOPP06PHj3q/dDo+ujvM5FmqKwsY/Pmt9lTUtDoO0zcnUh1GZHq4yQlpTT56ktaLzPj4osvZvHixTz33HP06tWr2aENuuIWabYdOwrYlVNAn85n0ym5Z4P2qag6xsHjWyg8vJrthSv5+OPXqKqq/Hx9UlIKPXsOAozS0oMcPVoUo+olFsyMyZMnM2fOHM4991w6d+7cosevN7jNLBX4K5ASbP+iu/80+CDg54GewGrgO+5eYWYpwLPAOOAAcKW7b23RqkVakcrKUjZteZusHqPJ6X5BndtVexWllfspqdjDrkMfsH3XSrZseZdIpIJOnXqQltaTnJx8unQZSVqnCfTtORxwDhzZzFtvP055ud7bJAwGDhzIjBkz+NnPfkZqampMHqMhV9zlwGR3LzGzZOBtM/sf4EfAg+7+vJk9BlwPPBp8PejuXzazGcA84MqYVC/SSuza9RGFBz6gd9oI0jpkANGhkIqqo1R7NQeOf8Kxin3sKf6I4oPb2bp1JYcP7yI393y+lP01Oqf1Aowh/S/gcEfoUllGh8R0qr2S0spiEhP1x3Fr179/f2bOnMnVV19Nbm5uTIe+GvJhwQ6UBIvJweTAZODbQfszwFyiwT01mAd4EXjYzMyb8hIzkZAoLy9hy9Z36NdzDP0SJ3C0vJCjFbvZe3gtkepydu78kCNH9rBr1zrcqz7fr3v3AeRknE/3joMBp6r6CCUVxRQdK6ToyHoAiou3N/iDhuXMS01N5Y477uC6665j4MCBLTKGXZ8G/Ro3s0SiwyFfBh4BNgOH3D0SbLIT6BfM9wN2ALh7xMwOEx1O2V/X8TMzM4lEIuwL00d6S4t76623SEtLa9LLyFuDI0e2UlTyAaW+kwPFWyku3saOHe9TWVlGVVUE92o6dkz5wj4lJbvYe3wV+ysKAOfTt7Zy8NAhPvtsK9u2rQain2NZO+yldUhOTiY/P58bb7yRK6644rQvUW9pDQpuj541o82sG/AyMLS5D2xms4BZEB0TevbZZ/n2t7+t8G6HEhISuPzyy/nRj37E448/Hu9ymiU5OZWEhCQikQqqqirq3f7w4eMcO1ZFTk5fwKmoKMXdee655ygoGMzChQuprm7am1lJbJgZkyZN4sYbb+Siiy6iS5c4vN+MuzdqAn4C3EH0CjopaDsXWBLMLwHODeaTgu3sdMccN26cV1dX+5YtW/yGG25wokMxmtr41KFDB7/ssst8xYoVfvz4cZcvqqio8A0bNvgTTzzhU6dO9W7dusX9OWvvU8+ePX3x4sV+6NChmD//48aNc68rh+ta4f8I6gygWzDfEXgLuARYCMwI2h8DbgrmZwOPBfMzgBfqe4ygQHd3P3r0qM+ePduzsrLi/iRpis3UuXNnv+yyy/ydd97xysrKWJ//bUJ1dbUXFBT4iy++6BMnTvQJEyZ4UlJS3J/L9jJlZmb697//ff/oo4/O2HPe3OA+G/gAWAOsBX4StA8G3gM2EQ3xlKA9NVjeFKwfXN9j1A7uGu+9954PHDgw7k+Yppabunbt6lOnTvXly5d7dXV17M74dqCiosJfffVV//nPf+5nnXWWp6Wlxf35bYtTZmam33rrrb5mzZoz/hw3K7jPxHSq4K65wpg9e7auLEI+denSxadPn+5vvPGGV1VVteCpLdXV1R6JRPzPf/6z//rXv/b+/ft7enp63J/zsE89e/b0UaNG+fr16+N2zoYyuGtUVFT4HXfc4R07doz7k6mpcVNqaqrPmDHDly9f7uXl5S1wKsvpVFVVeUlJiS9fvtznzp3rM2fO9LS0NF34NHBKTk729PR0v+WWW3zjxo1eWloa178MQx3c7u5lZWX+/vvve25urptZ3J9gTaefEhMTffr06f7OO+/on45xVFZW5rt37/aHH37Yv/vd73rHjh3181PH+Tpp0iT/wx/+4Hv27Gk1/3cJfXDXKCws9F/+8pe6gmil04gRI3z69Om6S6QVikQi/umnn/qCBQt82rRpnpGREffzJd5Tzfj1hg0bzshdIo3VZoLbPTp0ct999/mQIUPi/sRrik4jRozwRx55xAsLC5tweko8rFu3zv/2t7/53Xff7eedd56npKTE/Tw6E1OHDh180qRJ/tprr/mGDRvi/TScVpsK7hrbt2/3CRMmxP1EaK+TmXlubq4/9thjXlRU1OjnT1qPSCTir7/+ut9///1+1llneZcuXeJ+frX0lJ2d7fPmzfMlS5Z4JBKJ97e8QU4X3ObuxFteXp6vWrWqUfu4Ozt37uSpp57i3nvvpaysLEbVSW2JiYkMHDiQ22+/nauvvpr09HS9j3Qb4e5UV1ezfPlytmzZwttvv81rr70GwIEDB6ioqP+VoK1Jeno6F1xwAZdeeimXXHIJffr0CdW5mpeXx6pVq05ZcGiDu0Z1dTVPPPEEt99+O0eP6m0vYyUxMZHs7Gx++MMfcv3115OWlhaqHwJpvMrKSioro+8RvnDhQrZv387SpUtZvXo1lZWVlJeXx7nCkyUkJDBgwACuu+46Jk6cyMSJE+nQoUO8y2qSNh3cAJFIhJ07dzJz5kzefPPNlitMMDNyc3O54YYbuO666+jRo4cCux07evQopaWlfPjhh7z88svs27ePxYsXAzVvhhWfPElKSmL69OmcffbZzJo1q02cp20+uGsUFRUxZ84cFi1axJEjR1qgsjMjPz+fzMzML7S9+eabHDx4ME4VRY0YMYKZM2dy1VVXnVSfCEBZWRnbtm0DoufskiVL2LNnD+++++4ZefwxY8YwZMgQ7rjjDkaOHBnaq+tTaTfBDdFxuoULF3LvvfdSUFDQIsdsaYMGDWLgwIEMHz6ca665hqFDh9K9+z8+aNbdWbduXZ1DP0899RQbNmw45bry8nJWrlzZrCufESNGcOONNzJt2jSysrKafBxpnw4ePMiGDRvYv38/v/nNb4hEIqxcubLFxsh79Ojx+c/OpZdeSp8+fVrkuK1NuwruGocOHeL6669n0aJFLXrcpsrJyWHQoEHcdtttDB06lCFDhsTkcY4fP86yZctO+VagZWVlzJs3j9LS0jr3v/nmm7n88svJyMiISX3S/lRVVbFs2TLKysrYvn07jzzyCFVVVWzcuLHBb1mbmJhIbm4u+fn53HTTTZxzzjkxrjr+2mVwAxw+fJjf/e53zJ07l/376/wch5jo2rUr3bp147bbbqNz585861vfonfv3phZ3Mbeam4lOp141idtX805ePz4cV566SUqKip45JFHKCoqIhKJUFT0xQ9F7tWrF1lZWdx5551Mnz6d5OTkM/IJM61Buw1uiJ4ob7zxBtdccw179uyJ2ZvSJycnk5KSwtVXX02/fv248MILOfvss0lNTVUQitTB3SkvL6e6upqDBw/yzDPPUFFRwfz587nyyiu56aabGDBgACkpKe3u56hdBzdET44DBw4wZ84cnnzyyRY7bmJiIp06dWLGjBnk5+dzySWX0L17d5KTk1vsMUTaG3enuLiYrl27kpTUfj8k+XTB3S6+K2ZGr169eOihh+jevTsLFixg165dTT7eN77xjc//gZeYmEhOTk67+fNNJNbMjJ49e8a7jFatXQR3jbS0NO6//34uv/xy7rzzTlasWEFVVf0fwjps2DCys7OZM2cOKSkpjBw5kk6dOp2BikVETtaugrvGeeedx9KlS/nxj3/Mgw8+SCQS+cL6Xr160bdvX2677TZ69+5NXl4effv2jVO1IiJf1C6DGyAlJYV77rmHsWPHct9997F371569erFzTffzKhRoxg7dqzusBCRVqne4DazVOCvQEqw/Yvu/lMzexr4OnA42PS77l5g0aT7L2AKUBq0vx+L4psrOTmZK6+8kmnTpkXfccusXf73WkTCpSFX3OXAZHcvMbNk4G0z+59g3R3u/uIJ238TyA2mCcCjwddWycxITU2NdxkiIg1W760QwVvDlgSLycF0unsIpwLPBvutALqZmd7oQkSkhTToHjYzSzSzAqAIeN3d/x6susfM1pjZg2aWErT1A3bU2n1n0CYiIi2gQcHt7lXuPhroD4w3sxHA3cBQ4BygBzCnMQ9sZrPMbJWZrdq3b18jyxYRab8a9aoRdz8ELAMudvfdwXBIOfAUMD7YrBAYUGu3/kHbicea7+557p6nNzQSEWm4eoPbzDLMrFsw3xG4ENhQM24d3EUyDVgb7PIKcI1F5QOH3X13TKoXEWmHGnJXSSbwjJklEg36F9z9T2b2hpllAAYUAN8Ptn+V6K2Am4jeDjiz5csWEWm/6g1ud18DjDlF++Q6tndgdvNLExGRU9E7I4mIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCxtw93jVgZkeBT+JdR4z0AvbHu4gYaKv9grbbN/UrXAa5e8apViSd6Urq8Im758W7iFgws1VtsW9ttV/QdvumfrUdGioREQkZBbeISMi0luCeH+8CYqit9q2t9gvabt/UrzaiVfxzUkREGq61XHGLiEgDxT24zexiM/vEzDaZ2V3xrqexzOxJMysys7W12nqY2etmtjH42j1oNzN7KOjrGjMbG7/KT8/MBpjZMjNbb2brzOyWoD3UfTOzVDN7z8w+DPr1s6A9x8z+HtT/BzPrELSnBMubgvXZ8ay/PmaWaGYfmNmfguW20q+tZvaRmRWY2aqgLdTnYnPENbjNLBF4BPgmMAy4ysyGxbOmJngauPiEtruApe6eCywNliHaz9xgmgU8eoZqbIoI8G/uPgzIB2YHz03Y+1YOTHb3UcBo4GIzywfmAQ+6+5eBg8D1wfbXAweD9geD7VqzW4CPay23lX4BXODuo2vd+hf2c7Hp3D1uE3AusKTW8t3A3fGsqYn9yAbW1lr+BMgM5jOJ3qcO8P+Aq061XWufgD8CF7alvgGdgPeBCURfwJEUtH9+XgJLgHOD+aRgO4t37XX0pz/RAJsM/AmwttCvoMatQK8T2trMudjYKd5DJf2AHbWWdwZtYdfH3XcH83uAPsF8KPsb/Bk9Bvg7baBvwXBCAVAEvA5sBg65eyTYpHbtn/crWH8Y6HlmK26w/wTuBKqD5Z60jX4BOPCama02s1lBW+jPxaZqLa+cbLPc3c0stLfumFln4CXgVnc/Ymafrwtr39y9ChhtZt2Al4GhcS6p2czsEqDI3Veb2aR41xMDE9290Mx6A6+b2YbaK8N6LjZVvK+4C4EBtZb7B21ht9fMMgGCr0VBe6j6a2bJREP79+6+KGhuE30DcPdDwDKiQwjdzKzmQqZ27Z/3K1jfFThwhkttiK8Cl5nZVuB5osMl/0X4+wWAuxcGX4uI/rIdTxs6Fxsr3sG9EsgN/vPdAZgBvBLnmlrCK8C1wfy1RMeHa9qvCf7rnQ8crvWnXqti0Uvr3wIfu/sDtVaFum9mlhFcaWNmHYmO239MNMCnB5ud2K+a/k4H3vBg4LQ1cfe73b2/u2cT/Tl6w93/hZD3C8DM0swsvWYe+CdgLSE/F5sl3oPswBTgU6LjjP833vU0of4FwG6gkuhY2vVExwqXAhuBvwA9gm2N6F00m4GPgLx413+afk0kOq64BigIpilh7xtwNvBB0K+1wE+C9sHAe8AmYCGQErSnBsubgvWD492HBvRxEvCnttKvoA8fBtO6mpwI+7nYnEmvnBQRCZl4D5WIiEgjKbhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCZn/D0FstkREZS2+AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Aex7mcKr0J01",
        "outputId": "a36aaa35-ec20-4089-ddde-ab4742d3e90e"
      },
      "source": [
        "print(np.mean(test_total_reward))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-147.2620449863271\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "leyebGYRpqsF"
      },
      "source": [
        "Action list"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hGAH4YWDpp4u",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "24f547ee-6648-4d2a-f9c7-718b23e93251"
      },
      "source": [
        "print(\"Action list looks like \", action_list)\n",
        "print(\"Action list's shape looks like \", np.shape(action_list))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Action list looks like  [[2, 2, 2, 2, 2, 2, 1, 2, 1, 2, 1, 2, 2, 2, 2, 1, 1, 2, 1, 2, 1, 2, 1, 2, 2, 1, 2, 1, 2, 1, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 1, 2, 2, 2, 2, 2, 0, 3, 2, 3, 2, 3, 3, 2, 2, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 2, 3, 3, 2, 3, 2, 3, 3, 2, 2, 3, 3, 2, 2, 3, 3, 2, 2, 3, 2, 2, 3, 2, 2, 2, 2, 3, 2, 2, 2, 3, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 2, 1, 2, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 1, 2, 2, 2, 1, 1, 2, 2, 1, 1, 1, 2, 2, 1, 2, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 2, 2, 3, 2, 3, 3, 2, 3, 2, 3, 2, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3], [2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 1, 1, 2, 2, 1, 2, 1, 2, 1, 2, 2, 2, 1, 2, 1, 1, 2, 1, 2, 2, 2, 2, 2, 2, 1, 1, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 3, 3, 2, 3, 3, 3, 3, 3, 2, 3, 2, 3, 3, 3, 3, 2, 2, 3, 3, 3, 3, 2, 2, 2, 3, 3, 3, 2, 2, 3, 3, 3, 3, 2, 2, 3, 3, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 3, 2, 3, 2, 2, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1, 2, 1, 2, 2, 1, 2, 1, 2, 2, 2, 2, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 1, 2, 1, 2, 1, 1, 0, 2, 1, 1, 2, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 2, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 3, 2, 3, 2, 3, 3, 3, 3, 3, 2, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 2, 3, 3, 2, 2, 2, 3, 3, 2, 3, 0, 2, 3, 2, 0, 2, 3, 3, 2, 3, 2, 3, 2, 3, 2, 2, 3, 2, 3, 2, 2, 3, 3, 2, 3, 2, 2, 3, 2, 2, 2, 2, 3, 3, 2, 2, 2, 2, 3, 2, 2, 0, 1, 2, 1, 2, 1, 2, 1, 1, 1, 1, 2, 1, 2, 2, 1, 1, 1, 1, 2, 2, 1, 1, 1, 2, 1, 2, 1, 2, 2, 1, 1, 2, 2, 1, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 1, 2, 2, 2, 0, 2, 3, 2, 3, 3, 3, 2, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 2, 2, 3, 2, 3, 3, 3, 2, 3, 2, 3, 2, 2, 3, 2, 3, 2, 3, 2, 3, 3, 2, 2, 3, 2, 3, 3, 3, 2, 3, 3, 2, 2, 3, 2, 3, 2, 2, 3, 3, 2, 2, 2, 2, 2, 2, 0, 2, 1, 2, 2, 0, 1, 2, 2, 0, 2, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 2, 2, 1, 2, 2, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 2, 2, 2, 1, 2, 0, 2, 0, 3, 2, 3, 2, 0, 2, 0, 2, 3, 3, 3, 2, 2, 3, 2, 3, 2, 3, 2, 3, 3, 3, 2, 3, 3, 3, 3, 2, 2, 3, 2, 2, 2, 3, 2, 2, 3, 2, 3, 2, 2, 3, 2, 2, 3, 2, 0, 2, 3, 2, 2, 2, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 2, 2, 2, 2, 2, 2, 0, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 2, 1, 2, 2, 1, 2, 2, 1, 2, 1, 1, 2, 2, 0, 2, 1, 2, 1, 1, 1, 2, 1, 2, 2, 1, 2, 1, 2, 2, 2, 2, 1, 2, 2, 1, 2, 2, 1, 2, 1, 2, 2, 2, 1, 1, 2, 3, 3, 2, 3, 2, 3, 2, 3, 2, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 3, 2, 3, 3, 2, 2, 2, 2, 3, 2, 3, 2, 3, 3, 2, 3, 2, 2, 3, 2, 3, 2, 2, 2, 3, 2, 0, 2, 1, 2, 1, 2, 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 1, 1, 2, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 2, 1, 2, 1, 2, 2, 1, 2, 1, 2, 1, 2, 1, 2, 2, 2, 1, 1, 2, 2, 2, 1, 1, 2, 1, 2, 2, 1, 2, 1, 2, 1, 2, 1, 1, 2, 2, 1, 2, 2, 2, 0, 2, 2, 1, 2, 2, 2, 3, 0, 3, 2, 3, 3, 2, 3, 3, 2, 3, 3, 2, 2, 3, 3, 3, 3, 2, 3, 3, 3, 2, 2, 3, 2, 2, 3, 3, 3, 2, 2, 2, 3, 3, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 1, 2, 2, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 2, 2, 1, 2, 1, 1, 2, 2, 1, 2, 1, 1, 2, 2, 2, 1, 2, 1], [0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 1, 1, 2, 1, 2, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 2, 2, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 2, 3, 2, 3, 2, 3, 3, 2, 2, 3, 3, 2, 3, 2, 3, 3, 2, 3, 2, 3, 2, 2, 2, 2, 2, 2, 2, 3, 2, 3, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1], [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 2, 2, 3, 2, 3, 2, 2, 2, 2, 2, 2, 0, 2, 1, 2, 2, 0, 1, 2, 1, 2, 2, 1, 1, 2, 0, 2, 1, 2, 2, 2, 2, 2, 1, 1, 2, 1, 0, 2, 2, 2, 2, 1, 2, 2, 2, 1, 1, 2, 2, 2, 1, 2, 1, 2, 2, 2, 2, 1, 2, 1, 2, 1, 1, 2, 2, 1, 2, 2, 2, 3, 2, 3, 2, 2, 3, 2, 3, 2, 3, 3, 2, 3, 3, 3, 2, 3, 3, 3, 3, 2, 3, 3, 2, 2, 2, 2, 3, 2, 3, 3, 2, 3, 2, 2, 3, 2, 2, 3, 2, 2, 2, 3, 2, 2, 3, 2, 2, 3, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 0, 0, 2, 2, 1, 1, 2, 2, 2, 1, 1, 2, 2, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 0, 1, 2, 1, 1, 2, 1, 2, 2, 1, 2, 1, 2, 1, 2, 1, 1, 2, 2, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 2, 3, 3, 3, 2, 3, 3, 3, 2, 3, 3, 2, 2, 2, 3, 2, 3, 2, 2, 3, 3, 2, 3, 3, 3, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 2, 2, 1, 2, 1, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1], [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 2, 3, 3, 2, 3, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 2, 3, 2, 2, 2, 2, 3, 2, 2, 2, 2, 1, 1, 1, 2, 1, 2, 1, 2, 2, 1, 2, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 1, 1, 1, 2, 1, 2, 2, 1, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 1, 2, 2, 2, 3, 2, 2, 0, 2, 2, 2, 3, 2, 3, 2, 2, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 2, 3, 3, 3, 3, 2, 3, 3, 2, 3, 3, 3, 2, 2, 3, 2, 3, 3, 2, 3, 2, 3, 2, 2, 2, 3, 2, 3, 2, 3, 2, 3, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 2, 1, 1, 1, 2, 2, 1, 2, 1, 1, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]]\n",
            "Action list's shape looks like  (5,)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/numpy/core/_asarray.py:83: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
            "  return array(a, dtype, copy=False, order=order)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fNkmwucrHMen"
      },
      "source": [
        "Analysis of actions taken by agent"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WHdAItjj1nxw",
        "outputId": "20aea8b1-e011-4397-b775-7b4c4b593871"
      },
      "source": [
        "distribution = {}\n",
        "for actions in action_list:\n",
        "  for action in actions:\n",
        "    if action not in distribution.keys():\n",
        "      distribution[action] = 1\n",
        "    else:\n",
        "      distribution[action] += 1\n",
        "print(distribution)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{2: 1144, 1: 516, 0: 30, 3: 501}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ricE0schY75M"
      },
      "source": [
        "Saving the result of Model Testing\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GZsMkGmIY42b",
        "outputId": "c47a3123-eb1b-4dc1-f1b2-7a09a82cd8a6"
      },
      "source": [
        "PATH = \"Action_List.npy\" # Can be modified into the name or path you want\n",
        "np.save(PATH ,np.array(action_list)) "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:2: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
            "  \n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "asK7WfbkaLjt"
      },
      "source": [
        "### This is the file you need to submit !!!\n",
        "Download the testing result to your device\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "c-CqyhHzaWAL",
        "outputId": "38653c82-673e-4f90-8746-3a0424fe3aca"
      },
      "source": [
        "from google.colab import files\n",
        "files.download(PATH)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "download(\"download_899cef0d-01bc-40fd-a501-1573f2382641\", \"Action_List.npy\", 4689)"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "seT4NUmWmAZ1"
      },
      "source": [
        "# Server \n",
        "The code below simulate the environment on the judge server. Can be used for testing."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U69c-YTxaw6b",
        "outputId": "65dd4cf4-e667-469d-a7dc-34f6a053e1d0"
      },
      "source": [
        "action_list = np.load(PATH,allow_pickle=True) # The action list you upload\n",
        "seed = 543 # Do not revise this\n",
        "fix(env, seed)\n",
        "\n",
        "agent.network.eval()  # set network to evaluation mode\n",
        "\n",
        "test_total_reward = []\n",
        "if len(action_list) != 5:\n",
        "  print(\"Wrong format of file !!!\")\n",
        "  exit(0)\n",
        "for actions in action_list:\n",
        "  state = env.reset()\n",
        "  img = plt.imshow(env.render(mode='rgb_array'))\n",
        "\n",
        "  total_reward = 0\n",
        "\n",
        "  done = False\n",
        "\n",
        "  for action in actions:\n",
        "  \n",
        "      state, reward, done, _ = env.step(action)\n",
        "      total_reward += reward\n",
        "      if done:\n",
        "        break\n",
        "\n",
        "  print(f\"Your reward is : %.2f\"%total_reward)\n",
        "  test_total_reward.append(total_reward)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/__init__.py:422: UserWarning: torch.set_deterministic is deprecated and will be removed in a future release. Please use torch.use_deterministic_algorithms instead\n",
            "  \"torch.set_deterministic is deprecated and will be removed in a future \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Your reward is : -29.53\n",
            "Your reward is : -36.44\n",
            "Your reward is : -194.16\n",
            "Your reward is : -268.27\n",
            "Your reward is : -207.91\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAc+klEQVR4nO3de3RU9b338feXJCRcwv1iCAmgDQcRFSFC9NAWWbVaq8JxsRR7qBbx0Cqtlx4ROc9qS/sstVqrz3HJ0getFts+qHgry9qDFpDWWpWLEQGxIuUWgSg3wZDLZL7PH7NDRyD3mUx25vNaa6/Z+7f3zP7+MptPNr/Zk23ujoiIhEenVBcgIiLNo+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQSVpwm9nFZvaBmW0xszuStR8RkXRjybiO28wygL8DFwK7gNXA1e6+KeE7ExFJM8k64x4HbHH3re5eDTwFTE7SvkRE0kpmkl43H9gZt7wLGF/fxmamr29KQnXt2pvMzByqKj+jqrqCnJzudO8ygKyMbq163dpoFZ9Xfcrnn+8jKyuHrl17E4nUUFGxH/dogqoXiXF3O1l7soK7UWY2C5iVqv1Lx9Wjxylc8JVbGN7vYjaVv8DyFfczfvy3KR46k9zsvFa99oGj/2D1R4+xZs3TnF8yk7OHTONo5ABvblzI+vVLE9QDkYYla6ikDCiIWx4ctB3j7gvdvdjdi5NUg6Sp006bQH7PsThRKioOEI3WJnwf1dUV7Cx7h4OV2+nXdThDBo8jN3dAwvcjcjLJCu7VQJGZDTOzzsA0QKcjknS9euVTOKiYXjlD2H34Xbb+429UVx9Nyr527lxH2f61RKKVDOo5hmHDSpKyH5HjJSW43T0CfB9YBrwPPOPuG5OxL5E6mZmd+dJpX2Fwr3Oprv2cj/etY9eu0qTtr7q6gq3/+Bt7Dr9Lny6nMiT/XHr0OCVp+xOpk7Qxbnd/GXg5Wa8vcrz8/LMYMexiemTns2H3s6x751lqa2uA2Oc7tV5DJFrZqn1EveYLyzt2rOXjYaWckjuaQb3GMGRIMe+991Kr9iHSmJR9OCmSSBkZWZx22gTyuo9m/9GtbC97i0OHdgdrnX37trGl+zKMk35IfxwDTn6hkxPl00+3HVuORKr4aOvr5PU+m/we4xiSP45t297m8OHy1nZJpF4KbukQCgrOYVDv0WR2yubjg++wffvaL6zftOkVPvrorwnZV1XVkS8s79xZyu5T3+WU7mfTs9tgCgvHsHHjMuoLf5HWUnBLh1BdXcHH+96lIrKPHR+v5tChj7+wPhqNcPTooaTsOxKpYuOmP9I1py+fHf44ONNXaEvyJOUr780uQl/AkQTIzu5Obm5/Dh7cTSTSurHslujZcxAVFQepqalo831Lx1TfF3AU3CIi7VR9wa0/6yoiEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIh06q/x21m24DDQC0QcfdiM+sDPA0MBbYBV7r7gdaVKSIidRJxxn2Bu4929+Jg+Q5gubsXAcuDZRERSZBkDJVMBhYF84uAKUnYh4hI2mptcDvwipmtNbNZQdtAd6+7S+seYGAr9yEiInFae8/JCe5eZmYDgFfNbHP8Snf3+u5uEwT9rJOtExGR+iXs1mVmNh84AvwHMNHdd5tZHvCau/9LI8/VrctERI6T8FuXmVk3M8utmwe+DmwAlgLXBptdC/y+pfsQEZETtfiM28xOBV4IFjOB/+fud5pZX+AZoBDYTuxywP2NvJbOuEVEjqO7vIuIhIzu8i4i0kEouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyjQa3mT1uZuVmtiGurY+ZvWpmHwaPvYN2M7MHzWyLma03szHJLF5EJB015Yz718DFx7XdASx39yJgebAM8A2gKJhmAQ8npkwREanTaHC7+5+B/cc1TwYWBfOLgClx7U96zJtALzPLS1SxIiLS8jHuge6+O5jfAwwM5vOBnXHb7QraTmBms8xsjZmtaWENIiJpKbO1L+DubmbeguctBBYCtOT5IiLpqqVn3HvrhkCCx/KgvQwoiNtucNAmIiIJ0tLgXgpcG8xfC/w+rv2a4OqSEuBQ3JCKiIgkgLk3PEphZouBiUA/YC/wE+BF4BmgENgOXOnu+83MgIeIXYVSAcxw90bHsDVUIiJyIne3k7U3GtxtQcEtInKi+oJb35wUEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyDQa3Gb2uJmVm9mGuLb5ZlZmZqXBdEncunlmtsXMPjCzi5JVuIhIumrKzYK/AhwBnnT3UUHbfOCIu9933LYjgcXAOGAQ8CdguLvXNrIP3XNSROQ4Lb7npLv/GdjfxP1MBp5y9yp3/wewhViIi4hIgrRmjPv7ZrY+GErpHbTlAzvjttkVtJ3AzGaZ2RozW9OKGkRE0k5Lg/th4DRgNLAb+GVzX8DdF7p7sbsXt7AGEZG01KLgdve97l7r7lHgUf45HFIGFMRtOjhoExGRBGlRcJtZXtzivwF1V5wsBaaZWbaZDQOKgLdbV6KIiMTLbGwDM1sMTAT6mdku4CfARDMbDTiwDfgugLtvNLNngE1ABJjd2BUlIiLSPI1eDtgmRehyQBGRE7T4ckAREWlfFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIdNocJtZgZmtNLNNZrbRzG4O2vuY2atm9mHw2DtoNzN70My2mNl6MxuT7E6IiKSTppxxR4D/dPeRQAkw28xGAncAy929CFgeLAN8g9jd3YuAWcDDCa9aRCSNNRrc7r7b3dcF84eB94F8YDKwKNhsETAlmJ8MPOkxbwK9zCwv4ZWLiKSpZo1xm9lQ4BzgLWCgu+8OVu0BBgbz+cDOuKftCtqOf61ZZrbGzNY0s2YRkbTW5OA2s+7Ac8At7v5Z/Dp3d8Cbs2N3X+juxe5e3JzniYikuyYFt5llEQvt37n780Hz3rohkOCxPGgvAwrinj44aBMRkQRoylUlBvwKeN/d749btRS4Npi/Fvh9XPs1wdUlJcChuCEVERFpJYuNcjSwgdkE4C/Ae0A0aP4vYuPczwCFwHbgSnffHwT9Q8DFQAUww90bHMc2s2YNs4iIpAN3t5O1NxrcbUHBLSJyovqCW9+cFBEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMg05WbBBWa20sw2mdlGM7s5aJ9vZmVmVhpMl8Q9Z56ZbTGzD8zsomR2QEQk3TTlZsF5QJ67rzOzXGAtMAW4Ejji7vcdt/1IYDEwDhgE/AkY7u61DexD95wUETlOi+856e673X1dMH8YeB/Ib+Apk4Gn3L3K3f8BbCEW4iIikgDNGuM2s6HAOcBbQdP3zWy9mT1uZr2DtnxgZ9zTdtFw0IsAcNdd3+Wee2DUKBg5EgYNSnVFbW/ixIn8+tf/wiWXwBlnwIgRkJGR6qqkvcls6oZm1h14DrjF3T8zs4eB/w148PhL4LpmvN4sYFbzypWO7MwzTyUvDyZNii3v3g2bNsXm/+d/YMsWcIc9e6C23oG3cOvfvz/jxh3hjDNiy5EIvPEG1NTArl3w4oux9kOH4PDh1NUpqdWk4DazLGKh/Tt3fx7A3ffGrX8UeClYLAMK4p4+OGj7AndfCCwMnq8xbjnGglG9QYP+edZ9wQWx0K6thWXL4OjRWLD/9repqzOZ6n4GWVnw1a/G5t1h+vTY/IYN8MEHsfknn4S9e098Dem4mnJViQG/At539/vj2vPiNvs3YEMwvxSYZmbZZjYMKALeTlzJko6i0VhoRyJQUQGffx4L73RS94urthYqK2M/g88/j/1sJL005Yz7X4FvA++ZWWnQ9l/A1WY2mthQyTbguwDuvtHMngE2ARFgdkNXlIjEc49NEBsaKA2OuGXLYOvW2Lr9+zt+WNX9HCIRWLECqquhrAyWLo2tP3Ik/X5xyT81Gtzu/jpwsktSXm7gOXcCd7aiLklDR47AH/4QG/6IRmNjuJ98kuqq2l5pKTz6KGzfHvs57NjR8X9RSfM0+cNJkWTbsQPmz091Fal3//2wZk2qq5D2TF95FxEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkdB23iEg706lTJzp1qv+8WmfcIiLtRKdOnRgyZAg///nPGTVqVP3btWFNIiJSj06dOnHLLbewevVqbrvtNjIa+EPsGioREUmhzMxMvv71rzNv3jzGjh1Lly5dGn9OG9QlIiInUVxczP33309JSQlZWVlNfp6CW0SkjQ0ZMoQbb7yRq6++moKCgsafcBwFt4hIGzAzBg4cyM0338yMGTMYMGAAZie9iXujFNwiIknWuXNnLrnkEh599FH69OnT4KV+TaHgFhFJEjPjoosu4qabbuLLX/4y3bt3T8jrKrhFRBLMzJg0aRJz587lvPPOS1hg12k0uM0sB/gzkB1s/6y7/yS4EfBTQF9gLfBtd682s2zgSWAssA+4yt23JbRqEZF2qrCwkGnTpvHTn/6UnJycpOyjKWfcVcAkdz9iZlnA62b2R+CHwAPu/pSZPQLMBB4OHg+4+5fMbBpwD3BVUqoXEWknBg8ezIwZM5g+fTpFRUUt/uCxKZpys2AHjgSLWcHkwCTgW0H7ImA+seCeHMwDPAs8ZGYWvI6ISIeSk5PDnDlzuO666ygsLGz1B49N0aQxbjPLIDYc8iVgAfARcNDdI8Emu4D8YD4f2Ang7hEzO0RsOOXT+l4/Ly+PSCTCJ+l4S2855i9/+QvdunUjnX/Hb9u2jUOHDpGRkUFtbW2qy5EGZGVlUVJSwg033MCVV17Z4FfUE61Jwe3utcBoM+sFvACMaO2OzWwWMAtiY0JPPvkk3/rWtxTeaahTp05cccUV/PCHP+TRRx9NdTntwm9+8xtKS0tZsmQJkUgkrX+ZtTdmxsSJE7nhhhu46KKL6NGjR9sX4e7NmoAfA3OInUFnBm3nAcuC+WXAecF8ZrCdNfSaY8eO9Wg06lu3bvXrr7/eiQ3FaOrgU+fOnf3yyy/3N998048ePeryRdXV1b5582Z/7LHHfPLkyd6rV6+Uv2fpPvXt29dffPFFP3jwYNLf/7Fjx7rXl8P1rfB/BnV/oFcw3wX4C3ApsASYFrQ/AtwYzM8GHgnmpwHPNLaPoEB3dz98+LDPnj3bBw0alPI3SVNypu7du/vll1/ub7zxhtfU1CT7+O8QotGol5aW+rPPPusTJkzw8ePHe2ZmZsrfy3SZ8vLy/Hvf+56/9957bfaetza4zwLeAdYDG4AfB+2nAm8DW4iFeHbQnhMsbwnWn9rYPuKDu87bb7/thYWFKX/DNCVu6tmzp0+ePNlXrVrl0Wg0eUd8GqiurvaXX37Zf/azn/npp5/u3bp1S/n72xGnvLw8v+WWW3z9+vVt/h63KrjbYjpZcNedYcyePVtnFiGfevTo4VOnTvUVK1Z4bW1tAg9tiUajHolE/A9/+IP/4he/8MGDB3tubm7K3/OwT3379vWzzz7bN23alLJjNpTBXae6utrnzJnjXbp0Sfmbqal5U05Ojk+bNs1XrVrlVVVVCTiUpSG1tbV+5MgRX7Vqlc+fP99nzJjh3bp104lPE6esrCzPzc31m2++2T/88EOvqKhI6f8MQx3c7u6VlZW+bt06LyoqcjNL+RusqeEpIyPDp06d6m+88YY+dEyhyspK3717tz/00EP+ne98x7t06aJ/P/UcrxMnTvSnn37a9+zZ024+dwl9cNcpKyvzu+66S2cQ7XQaNWqUT506VVeJtEORSMT//ve/++LFi33KlCnev3//lB8vqZ7qxq83b97cJleJNFeHCW732NDJvffe68OHD0/5G68pNo0aNcoXLFjgZWVlLTg8JRU2btzof/3rX33evHl+/vnne3Z2dsqPo7aYOnfu7BMnTvRXXnnFN2/enOq3oUEdKrjr7Nixw8ePH5/yAyFdJzPzoqIif+SRR7y8vLzZ75+0H5FIxF999VW/7777/PTTT/cePXqk/PhK9DR06FC/5557fNmyZR6JRFL9I2+ShoLb3J1UKy4u9jVr1jTrOe7Orl27eOKJJ7j77ruprKxMUnUSLyMjg8LCQm677TamT59Obm5uUv+YjrQddycajbJq1Sq2bt3K66+/ziuvvALAvn37qK6uTnGFzZObm8sFF1zAZZddxqWXXsrAgQNDdawWFxezZs2akxYc2uCuE41Geeyxx7jttts4fPhwgiuTOhkZGQwdOpQf/OAHzJw5k27duoXqH4E0X01NDTU1NQAsWbKEHTt2sHz5ctauXUtNTQ1VVVUprvBEnTp1oqCggOuuu44JEyYwYcIEOnfunOqyWqRDBzdAJBJh165dzJgxg9deey1xhQlmRlFREddffz3XXXcdffr0UWCnscOHD1NRUcG7777LCy+8wCeffMKLL74IxE6iUpUnmZmZTJ06lbPOOotZs2Z1iOO0wwd3nfLycubOncvzzz/PZ599loDK2kZJSQl5eXlfaHvttdc4cOBAiiqKGTVqFDNmzODqq68+oT4RgMrKSrZv3w7Ejtlly5axZ88e/va3v7XJ/s855xyGDx/OnDlzOPPMM0N7dn0yaRPcEBunW7JkCXfffTelpaUJec1EGzJkCIWFhZxxxhlcc801jBgxgt69ex9b7+5s3Lix3qGfJ554gs2bN590XVVVFatXr27Vmc+oUaO44YYbmDJlCoMGDWrx60h6OnDgAJs3b+bTTz/ll7/8JZFIhNWrVydsjLxPnz7H/u1cdtllDBw4MCGv296kVXDXOXjwIDNnzuT5559P6Ou21LBhwxgyZAi33norI0aMYPjw4UnZz9GjR1m5ciXRaPSEdZWVldxzzz1UVFTU+/ybbrqJK664gv79+yelPkk/tbW1rFy5ksrKSnbs2MGCBQuora3lww8/POlxejIZGRkUFRVRUlLCjTfeyLnnnpvkqlMvLYMb4NChQ/z2t79l/vz5fPppvfdxSIqePXvSq1cvbr31Vrp37843v/lNBgwYgJmlbOyt7lKihqSyPun46o7Bo0eP8txzz1FdXc2CBQsoLy8nEolQXl7+he379evHoEGDuP3225k6dSpZWVltcoeZ9iBtgxtiB8qKFSu45ppr2LNnT5N/wzdXVlYW2dnZTJ8+nfz8fC688ELOOusscnJyFIQi9XB3qqqqiEajHDhwgEWLFlFdXc3ChQu56qqruPHGGykoKCA7Ozvt/h2ldXBD7ODYt28fc+fO5fHHH0/Y62ZkZNC1a1emTZtGSUkJl156Kb179yYrKyth+xBJN+7O/v376dmzJ5mZTbpJV4fUUHCnxU/FzOjXrx8PPvggvXv3ZvHixXz88cctfr2vfe1rxz7Ay8jIYNiwYWnz3zeRZDMz+vbtm+oy2rW0CO463bp147777uOKK67g9ttv580332zSDVlHjhzJ0KFDmTt3LtnZ2Zx55pl07dq1DSoWETlRWgV3nfPPP5/ly5fzox/9iAceeIBIJPKF9f369eOUU07h1ltvZcCAARQXF3PKKaekqFoRkS9Ky+AGyM7O5s4772TMmDHce++97N27l379+nHTTTdx9tlnM2bMGF1hISLtUqPBbWY5wJ+B7GD7Z939J2b2a+CrwKFg0++4e6nFku6/gUuAiqB9XTKKb62srCyuuuoqpkyZEvuLW2Zp+em1iIRLU864q4BJ7n7EzLKA183sj8G6Oe7+7HHbfwMoCqbxwMPBY7tkZuTk5KS6DBGRJmv0UojgT8MeCRazgqmhawgnA08Gz3sT6GVm+kMXIiIJ0qRr2Mwsw8xKgXLgVXd/K1h1p5mtN7MHzCw7aMsHdsY9fVfQJiIiCdCk4Hb3WncfDQwGxpnZKGAeMAI4F+gDzG3Ojs1slpmtMbM1n3zySTPLFhFJX8361oi7HwRWAhe7++5gOKQKeAIYF2xWBhTEPW1w0Hb8ay1092J3L9YfNBIRabpGg9vM+ptZr2C+C3AhsLlu3Dq4imQKsCF4ylLgGospAQ65++6kVC8ikoaaclVJHrDIzDKIBf0z7v6Sma0ws/6AAaXA94LtXyZ2KeAWYpcDzkh82SIi6avR4Hb39cA5J2mfVM/2DsxufWkiInIy+stIIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZAxd091DZjZYeCDVNeRJP2AT1NdRBJ01H5Bx+2b+hUuQ9y9/8lWZLZ1JfX4wN2LU11EMpjZmo7Yt47aL+i4fVO/Og4NlYiIhIyCW0QkZNpLcC9MdQFJ1FH71lH7BR23b+pXB9EuPpwUEZGmay9n3CIi0kQpD24zu9jMPjCzLWZ2R6rraS4ze9zMys1sQ1xbHzN71cw+DB57B+1mZg8GfV1vZmNSV3nDzKzAzFaa2SYz22hmNwftoe6bmeWY2dtm9m7Qr58G7cPM7K2g/qfNrHPQnh0sbwnWD01l/Y0xswwze8fMXgqWO0q/tpnZe2ZWamZrgrZQH4utkdLgNrMMYAHwDWAkcLWZjUxlTS3wa+Di49ruAJa7exGwPFiGWD+LgmkW8HAb1dgSEeA/3X0kUALMDt6bsPetCpjk7mcDo4GLzawEuAd4wN2/BBwAZgbbzwQOBO0PBNu1ZzcD78ctd5R+AVzg7qPjLv0L+7HYcu6esgk4D1gWtzwPmJfKmlrYj6HAhrjlD4C8YD6P2HXqAP8XuPpk27X3Cfg9cGFH6hvQFVgHjCf2BY7MoP3YcQksA84L5jOD7SzVtdfTn8HEAmwS8BJgHaFfQY3bgH7HtXWYY7G5U6qHSvKBnXHLu4K2sBvo7ruD+T3AwGA+lP0N/ht9DvAWHaBvwXBCKVAOvAp8BBx090iwSXztx/oVrD8E9G3bipvs/wC3A9FguS8do18ADrxiZmvNbFbQFvpjsaXayzcnOyx3dzML7aU7ZtYdeA64xd0/M7Nj68LaN3evBUabWS/gBWBEiktqNTO7FCh397VmNjHV9STBBHcvM7MBwKtmtjl+ZViPxZZK9Rl3GVAQtzw4aAu7vWaWBxA8lgftoeqvmWURC+3fufvzQXOH6BuAux8EVhIbQuhlZnUnMvG1H+tXsL4nsK+NS22KfwUuN7NtwFPEhkv+m/D3CwB3Lwsey4n9sh1HBzoWmyvVwb0aKAo++e4MTAOWprimRFgKXBvMX0tsfLiu/ZrgU+8S4FDcf/XaFYudWv8KeN/d749bFeq+mVn/4EwbM+tCbNz+fWIBPjXY7Ph+1fV3KrDCg4HT9sTd57n7YHcfSuzf0Qp3/3dC3i8AM+tmZrl188DXgQ2E/FhslVQPsgOXAH8nNs74v1JdTwvqXwzsBmqIjaXNJDZWuBz4EPgT0CfY1ohdRfMR8B5QnOr6G+jXBGLjiuuB0mC6JOx9A84C3gn6tQH4cdB+KvA2sAVYAmQH7TnB8pZg/amp7kMT+jgReKmj9Cvow7vBtLEuJ8J+LLZm0jcnRURCJtVDJSIi0kwKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURC5v8Dix/mJUV1WnwAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TjFBWwQP1hVe"
      },
      "source": [
        "# Your score"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GpJpZz3Wbm0X",
        "outputId": "7d4677c7-b285-42d3-d8c0-5f0a0d8230c8"
      },
      "source": [
        "print(f\"Your final reward is : %.2f\"%np.mean(test_total_reward))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Your final reward is : -147.26\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wUBtYXG2eaqf"
      },
      "source": [
        "## Reference\n",
        "\n",
        "Below are some useful tips for you to get high score.\n",
        "\n",
        "- [DRL Lecture 1: Policy Gradient (Review)](https://youtu.be/z95ZYgPgXOY)\n",
        "- [ML Lecture 23-3: Reinforcement Learning (including Q-learning) start at 30:00](https://youtu.be/2-JNBzCq77c?t=1800)\n",
        "- [Lecture 7: Policy Gradient, David Silver](http://www0.cs.ucl.ac.uk/staff/d.silver/web/Teaching_files/pg.pdf)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eZ7VDw-C19Qe"
      },
      "source": [
        ""
      ]
    }
  ]
}